hexdag 0.5.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. hexdag/__init__.py +116 -0
  2. hexdag/__main__.py +30 -0
  3. hexdag/adapters/executors/__init__.py +5 -0
  4. hexdag/adapters/executors/local_executor.py +316 -0
  5. hexdag/builtin/__init__.py +6 -0
  6. hexdag/builtin/adapters/__init__.py +51 -0
  7. hexdag/builtin/adapters/anthropic/__init__.py +5 -0
  8. hexdag/builtin/adapters/anthropic/anthropic_adapter.py +151 -0
  9. hexdag/builtin/adapters/database/__init__.py +6 -0
  10. hexdag/builtin/adapters/database/csv/csv_adapter.py +249 -0
  11. hexdag/builtin/adapters/database/pgvector/__init__.py +5 -0
  12. hexdag/builtin/adapters/database/pgvector/pgvector_adapter.py +478 -0
  13. hexdag/builtin/adapters/database/sqlalchemy/sqlalchemy_adapter.py +252 -0
  14. hexdag/builtin/adapters/database/sqlite/__init__.py +5 -0
  15. hexdag/builtin/adapters/database/sqlite/sqlite_adapter.py +410 -0
  16. hexdag/builtin/adapters/local/README.md +59 -0
  17. hexdag/builtin/adapters/local/__init__.py +7 -0
  18. hexdag/builtin/adapters/local/local_observer_manager.py +696 -0
  19. hexdag/builtin/adapters/memory/__init__.py +47 -0
  20. hexdag/builtin/adapters/memory/file_memory_adapter.py +297 -0
  21. hexdag/builtin/adapters/memory/in_memory_memory.py +216 -0
  22. hexdag/builtin/adapters/memory/schemas.py +57 -0
  23. hexdag/builtin/adapters/memory/session_memory.py +178 -0
  24. hexdag/builtin/adapters/memory/sqlite_memory_adapter.py +215 -0
  25. hexdag/builtin/adapters/memory/state_memory.py +280 -0
  26. hexdag/builtin/adapters/mock/README.md +89 -0
  27. hexdag/builtin/adapters/mock/__init__.py +15 -0
  28. hexdag/builtin/adapters/mock/hexdag.toml +50 -0
  29. hexdag/builtin/adapters/mock/mock_database.py +225 -0
  30. hexdag/builtin/adapters/mock/mock_embedding.py +223 -0
  31. hexdag/builtin/adapters/mock/mock_llm.py +177 -0
  32. hexdag/builtin/adapters/mock/mock_tool_adapter.py +192 -0
  33. hexdag/builtin/adapters/mock/mock_tool_router.py +232 -0
  34. hexdag/builtin/adapters/openai/__init__.py +5 -0
  35. hexdag/builtin/adapters/openai/openai_adapter.py +634 -0
  36. hexdag/builtin/adapters/secret/__init__.py +7 -0
  37. hexdag/builtin/adapters/secret/local_secret_adapter.py +248 -0
  38. hexdag/builtin/adapters/unified_tool_router.py +280 -0
  39. hexdag/builtin/macros/__init__.py +17 -0
  40. hexdag/builtin/macros/conversation_agent.py +390 -0
  41. hexdag/builtin/macros/llm_macro.py +151 -0
  42. hexdag/builtin/macros/reasoning_agent.py +423 -0
  43. hexdag/builtin/macros/tool_macro.py +380 -0
  44. hexdag/builtin/nodes/__init__.py +38 -0
  45. hexdag/builtin/nodes/_discovery.py +123 -0
  46. hexdag/builtin/nodes/agent_node.py +696 -0
  47. hexdag/builtin/nodes/base_node_factory.py +242 -0
  48. hexdag/builtin/nodes/composite_node.py +926 -0
  49. hexdag/builtin/nodes/data_node.py +201 -0
  50. hexdag/builtin/nodes/expression_node.py +487 -0
  51. hexdag/builtin/nodes/function_node.py +454 -0
  52. hexdag/builtin/nodes/llm_node.py +491 -0
  53. hexdag/builtin/nodes/loop_node.py +920 -0
  54. hexdag/builtin/nodes/mapped_input.py +518 -0
  55. hexdag/builtin/nodes/port_call_node.py +269 -0
  56. hexdag/builtin/nodes/tool_call_node.py +195 -0
  57. hexdag/builtin/nodes/tool_utils.py +390 -0
  58. hexdag/builtin/prompts/__init__.py +68 -0
  59. hexdag/builtin/prompts/base.py +422 -0
  60. hexdag/builtin/prompts/chat_prompts.py +303 -0
  61. hexdag/builtin/prompts/error_correction_prompts.py +320 -0
  62. hexdag/builtin/prompts/tool_prompts.py +160 -0
  63. hexdag/builtin/tools/builtin_tools.py +84 -0
  64. hexdag/builtin/tools/database_tools.py +164 -0
  65. hexdag/cli/__init__.py +17 -0
  66. hexdag/cli/__main__.py +7 -0
  67. hexdag/cli/commands/__init__.py +27 -0
  68. hexdag/cli/commands/build_cmd.py +812 -0
  69. hexdag/cli/commands/create_cmd.py +208 -0
  70. hexdag/cli/commands/docs_cmd.py +293 -0
  71. hexdag/cli/commands/generate_types_cmd.py +252 -0
  72. hexdag/cli/commands/init_cmd.py +188 -0
  73. hexdag/cli/commands/pipeline_cmd.py +494 -0
  74. hexdag/cli/commands/plugin_dev_cmd.py +529 -0
  75. hexdag/cli/commands/plugins_cmd.py +441 -0
  76. hexdag/cli/commands/studio_cmd.py +101 -0
  77. hexdag/cli/commands/validate_cmd.py +221 -0
  78. hexdag/cli/main.py +84 -0
  79. hexdag/core/__init__.py +83 -0
  80. hexdag/core/config/__init__.py +20 -0
  81. hexdag/core/config/loader.py +479 -0
  82. hexdag/core/config/models.py +150 -0
  83. hexdag/core/configurable.py +294 -0
  84. hexdag/core/context/__init__.py +37 -0
  85. hexdag/core/context/execution_context.py +378 -0
  86. hexdag/core/docs/__init__.py +26 -0
  87. hexdag/core/docs/extractors.py +678 -0
  88. hexdag/core/docs/generators.py +890 -0
  89. hexdag/core/docs/models.py +120 -0
  90. hexdag/core/domain/__init__.py +10 -0
  91. hexdag/core/domain/dag.py +1225 -0
  92. hexdag/core/exceptions.py +234 -0
  93. hexdag/core/expression_parser.py +569 -0
  94. hexdag/core/logging.py +449 -0
  95. hexdag/core/models/__init__.py +17 -0
  96. hexdag/core/models/base.py +138 -0
  97. hexdag/core/orchestration/__init__.py +46 -0
  98. hexdag/core/orchestration/body_executor.py +481 -0
  99. hexdag/core/orchestration/components/__init__.py +97 -0
  100. hexdag/core/orchestration/components/adapter_lifecycle_manager.py +113 -0
  101. hexdag/core/orchestration/components/checkpoint_manager.py +134 -0
  102. hexdag/core/orchestration/components/execution_coordinator.py +360 -0
  103. hexdag/core/orchestration/components/health_check_manager.py +176 -0
  104. hexdag/core/orchestration/components/input_mapper.py +143 -0
  105. hexdag/core/orchestration/components/lifecycle_manager.py +583 -0
  106. hexdag/core/orchestration/components/node_executor.py +377 -0
  107. hexdag/core/orchestration/components/secret_manager.py +202 -0
  108. hexdag/core/orchestration/components/wave_executor.py +158 -0
  109. hexdag/core/orchestration/constants.py +17 -0
  110. hexdag/core/orchestration/events/README.md +312 -0
  111. hexdag/core/orchestration/events/__init__.py +104 -0
  112. hexdag/core/orchestration/events/batching.py +330 -0
  113. hexdag/core/orchestration/events/decorators.py +139 -0
  114. hexdag/core/orchestration/events/events.py +573 -0
  115. hexdag/core/orchestration/events/observers/__init__.py +30 -0
  116. hexdag/core/orchestration/events/observers/core_observers.py +690 -0
  117. hexdag/core/orchestration/events/observers/models.py +111 -0
  118. hexdag/core/orchestration/events/taxonomy.py +269 -0
  119. hexdag/core/orchestration/hook_context.py +237 -0
  120. hexdag/core/orchestration/hooks.py +437 -0
  121. hexdag/core/orchestration/models.py +418 -0
  122. hexdag/core/orchestration/orchestrator.py +910 -0
  123. hexdag/core/orchestration/orchestrator_factory.py +275 -0
  124. hexdag/core/orchestration/port_wrappers.py +327 -0
  125. hexdag/core/orchestration/prompt/__init__.py +32 -0
  126. hexdag/core/orchestration/prompt/template.py +332 -0
  127. hexdag/core/pipeline_builder/__init__.py +21 -0
  128. hexdag/core/pipeline_builder/component_instantiator.py +386 -0
  129. hexdag/core/pipeline_builder/include_tag.py +265 -0
  130. hexdag/core/pipeline_builder/pipeline_config.py +133 -0
  131. hexdag/core/pipeline_builder/py_tag.py +223 -0
  132. hexdag/core/pipeline_builder/tag_discovery.py +268 -0
  133. hexdag/core/pipeline_builder/yaml_builder.py +1196 -0
  134. hexdag/core/pipeline_builder/yaml_validator.py +569 -0
  135. hexdag/core/ports/__init__.py +65 -0
  136. hexdag/core/ports/api_call.py +133 -0
  137. hexdag/core/ports/database.py +489 -0
  138. hexdag/core/ports/embedding.py +215 -0
  139. hexdag/core/ports/executor.py +237 -0
  140. hexdag/core/ports/file_storage.py +117 -0
  141. hexdag/core/ports/healthcheck.py +87 -0
  142. hexdag/core/ports/llm.py +551 -0
  143. hexdag/core/ports/memory.py +70 -0
  144. hexdag/core/ports/observer_manager.py +130 -0
  145. hexdag/core/ports/secret.py +145 -0
  146. hexdag/core/ports/tool_router.py +94 -0
  147. hexdag/core/ports_builder.py +623 -0
  148. hexdag/core/protocols.py +273 -0
  149. hexdag/core/resolver.py +304 -0
  150. hexdag/core/schema/__init__.py +9 -0
  151. hexdag/core/schema/generator.py +742 -0
  152. hexdag/core/secrets.py +242 -0
  153. hexdag/core/types.py +413 -0
  154. hexdag/core/utils/async_warnings.py +206 -0
  155. hexdag/core/utils/schema_conversion.py +78 -0
  156. hexdag/core/utils/sql_validation.py +86 -0
  157. hexdag/core/validation/secure_json.py +148 -0
  158. hexdag/core/yaml_macro.py +517 -0
  159. hexdag/mcp_server.py +3120 -0
  160. hexdag/studio/__init__.py +10 -0
  161. hexdag/studio/build_ui.py +92 -0
  162. hexdag/studio/server/__init__.py +1 -0
  163. hexdag/studio/server/main.py +100 -0
  164. hexdag/studio/server/routes/__init__.py +9 -0
  165. hexdag/studio/server/routes/execute.py +208 -0
  166. hexdag/studio/server/routes/export.py +558 -0
  167. hexdag/studio/server/routes/files.py +207 -0
  168. hexdag/studio/server/routes/plugins.py +419 -0
  169. hexdag/studio/server/routes/validate.py +220 -0
  170. hexdag/studio/ui/index.html +13 -0
  171. hexdag/studio/ui/package-lock.json +2992 -0
  172. hexdag/studio/ui/package.json +31 -0
  173. hexdag/studio/ui/postcss.config.js +6 -0
  174. hexdag/studio/ui/public/hexdag.svg +5 -0
  175. hexdag/studio/ui/src/App.tsx +251 -0
  176. hexdag/studio/ui/src/components/Canvas.tsx +408 -0
  177. hexdag/studio/ui/src/components/ContextMenu.tsx +187 -0
  178. hexdag/studio/ui/src/components/FileBrowser.tsx +123 -0
  179. hexdag/studio/ui/src/components/Header.tsx +181 -0
  180. hexdag/studio/ui/src/components/HexdagNode.tsx +193 -0
  181. hexdag/studio/ui/src/components/NodeInspector.tsx +512 -0
  182. hexdag/studio/ui/src/components/NodePalette.tsx +262 -0
  183. hexdag/studio/ui/src/components/NodePortsSection.tsx +403 -0
  184. hexdag/studio/ui/src/components/PluginManager.tsx +347 -0
  185. hexdag/studio/ui/src/components/PortsEditor.tsx +481 -0
  186. hexdag/studio/ui/src/components/PythonEditor.tsx +195 -0
  187. hexdag/studio/ui/src/components/ValidationPanel.tsx +105 -0
  188. hexdag/studio/ui/src/components/YamlEditor.tsx +196 -0
  189. hexdag/studio/ui/src/components/index.ts +8 -0
  190. hexdag/studio/ui/src/index.css +92 -0
  191. hexdag/studio/ui/src/main.tsx +10 -0
  192. hexdag/studio/ui/src/types/index.ts +123 -0
  193. hexdag/studio/ui/src/vite-env.d.ts +1 -0
  194. hexdag/studio/ui/tailwind.config.js +29 -0
  195. hexdag/studio/ui/tsconfig.json +37 -0
  196. hexdag/studio/ui/tsconfig.node.json +13 -0
  197. hexdag/studio/ui/vite.config.ts +35 -0
  198. hexdag/visualization/__init__.py +69 -0
  199. hexdag/visualization/dag_visualizer.py +1020 -0
  200. hexdag-0.5.0.dev1.dist-info/METADATA +369 -0
  201. hexdag-0.5.0.dev1.dist-info/RECORD +261 -0
  202. hexdag-0.5.0.dev1.dist-info/WHEEL +4 -0
  203. hexdag-0.5.0.dev1.dist-info/entry_points.txt +4 -0
  204. hexdag-0.5.0.dev1.dist-info/licenses/LICENSE +190 -0
  205. hexdag_plugins/.gitignore +43 -0
  206. hexdag_plugins/README.md +73 -0
  207. hexdag_plugins/__init__.py +1 -0
  208. hexdag_plugins/azure/LICENSE +21 -0
  209. hexdag_plugins/azure/README.md +414 -0
  210. hexdag_plugins/azure/__init__.py +21 -0
  211. hexdag_plugins/azure/azure_blob_adapter.py +450 -0
  212. hexdag_plugins/azure/azure_cosmos_adapter.py +383 -0
  213. hexdag_plugins/azure/azure_keyvault_adapter.py +314 -0
  214. hexdag_plugins/azure/azure_openai_adapter.py +415 -0
  215. hexdag_plugins/azure/pyproject.toml +107 -0
  216. hexdag_plugins/azure/tests/__init__.py +1 -0
  217. hexdag_plugins/azure/tests/test_azure_blob_adapter.py +350 -0
  218. hexdag_plugins/azure/tests/test_azure_cosmos_adapter.py +323 -0
  219. hexdag_plugins/azure/tests/test_azure_keyvault_adapter.py +330 -0
  220. hexdag_plugins/azure/tests/test_azure_openai_adapter.py +329 -0
  221. hexdag_plugins/hexdag_etl/README.md +168 -0
  222. hexdag_plugins/hexdag_etl/__init__.py +53 -0
  223. hexdag_plugins/hexdag_etl/examples/01_simple_pandas_transform.py +270 -0
  224. hexdag_plugins/hexdag_etl/examples/02_simple_pandas_only.py +149 -0
  225. hexdag_plugins/hexdag_etl/examples/03_file_io_pipeline.py +109 -0
  226. hexdag_plugins/hexdag_etl/examples/test_pandas_transform.py +84 -0
  227. hexdag_plugins/hexdag_etl/hexdag.toml +25 -0
  228. hexdag_plugins/hexdag_etl/hexdag_etl/__init__.py +48 -0
  229. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/__init__.py +13 -0
  230. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/api_extract.py +230 -0
  231. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/base_node_factory.py +181 -0
  232. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/file_io.py +415 -0
  233. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/outlook.py +492 -0
  234. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/pandas_transform.py +563 -0
  235. hexdag_plugins/hexdag_etl/hexdag_etl/nodes/sql_extract_load.py +112 -0
  236. hexdag_plugins/hexdag_etl/pyproject.toml +82 -0
  237. hexdag_plugins/hexdag_etl/test_transform.py +54 -0
  238. hexdag_plugins/hexdag_etl/tests/test_plugin_integration.py +62 -0
  239. hexdag_plugins/mysql_adapter/LICENSE +21 -0
  240. hexdag_plugins/mysql_adapter/README.md +224 -0
  241. hexdag_plugins/mysql_adapter/__init__.py +6 -0
  242. hexdag_plugins/mysql_adapter/mysql_adapter.py +408 -0
  243. hexdag_plugins/mysql_adapter/pyproject.toml +93 -0
  244. hexdag_plugins/mysql_adapter/tests/test_mysql_adapter.py +259 -0
  245. hexdag_plugins/storage/README.md +184 -0
  246. hexdag_plugins/storage/__init__.py +19 -0
  247. hexdag_plugins/storage/file/__init__.py +5 -0
  248. hexdag_plugins/storage/file/local.py +325 -0
  249. hexdag_plugins/storage/ports/__init__.py +5 -0
  250. hexdag_plugins/storage/ports/vector_store.py +236 -0
  251. hexdag_plugins/storage/sql/__init__.py +7 -0
  252. hexdag_plugins/storage/sql/base.py +187 -0
  253. hexdag_plugins/storage/sql/mysql.py +27 -0
  254. hexdag_plugins/storage/sql/postgresql.py +27 -0
  255. hexdag_plugins/storage/tests/__init__.py +1 -0
  256. hexdag_plugins/storage/tests/test_local_file_storage.py +161 -0
  257. hexdag_plugins/storage/tests/test_sql_adapters.py +212 -0
  258. hexdag_plugins/storage/vector/__init__.py +7 -0
  259. hexdag_plugins/storage/vector/chromadb.py +223 -0
  260. hexdag_plugins/storage/vector/in_memory.py +285 -0
  261. hexdag_plugins/storage/vector/pgvector.py +502 -0
@@ -0,0 +1,890 @@
1
+ """Guide generators for MCP documentation.
2
+
3
+ This module generates documentation directly from extracted component
4
+ documentation - no external templates needed.
5
+ """
6
+
7
+ import json
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from hexdag.core.docs.models import AdapterDoc, NodeDoc, ToolDoc
12
+ from hexdag.core.logging import get_logger
13
+
14
+ logger = get_logger(__name__)
15
+
16
+ # Path to the generated schema
17
+ SCHEMA_PATH = Path(__file__).parent.parent.parent.parent / "schemas" / "pipeline-schema.json"
18
+
19
+
20
+ class GuideGenerator:
21
+ """Generate documentation guides from extracted component docs.
22
+
23
+ All documentation is generated programmatically from code introspection,
24
+ ensuring it stays in sync with the actual implementation.
25
+ """
26
+
27
+ def generate_adapter_guide(self, adapters: list[AdapterDoc]) -> str:
28
+ """Generate adapter creation guide.
29
+
30
+ Parameters
31
+ ----------
32
+ adapters : list[AdapterDoc]
33
+ List of adapter documentation objects
34
+
35
+ Returns
36
+ -------
37
+ str
38
+ Complete adapter guide as markdown
39
+ """
40
+ # Group adapters by port type
41
+ adapters_by_port: dict[str, list[AdapterDoc]] = {}
42
+ for adapter in adapters:
43
+ port = adapter.port_type
44
+ if port not in adapters_by_port:
45
+ adapters_by_port[port] = []
46
+ adapters_by_port[port].append(adapter)
47
+
48
+ lines = [
49
+ "# Creating Custom Adapters in hexDAG",
50
+ "",
51
+ "## Overview",
52
+ "",
53
+ "hexDAG uses adapters to connect pipelines to external services like LLMs,",
54
+ "databases, and APIs. Adapters implement ports (interfaces) with async methods.",
55
+ "",
56
+ "## Quick Start",
57
+ "",
58
+ "### Simple Adapter (No Secrets)",
59
+ "",
60
+ "```python",
61
+ "class MemoryCacheAdapter:",
62
+ ' """Simple in-memory cache adapter."""',
63
+ "",
64
+ " def __init__(self, max_size: int = 100, ttl: int = 3600):",
65
+ " self.cache = {}",
66
+ " self.max_size = max_size",
67
+ " self.ttl = ttl",
68
+ "",
69
+ " async def aget(self, key: str):",
70
+ " return self.cache.get(key)",
71
+ "",
72
+ " async def aset(self, key: str, value: any):",
73
+ " self.cache[key] = value",
74
+ "```",
75
+ "",
76
+ "### Adapter with Secrets",
77
+ "",
78
+ "Use `secret()` in defaults to declare secrets that auto-resolve from environment:",
79
+ "",
80
+ "```python",
81
+ "from hexdag.core.secrets import secret",
82
+ "",
83
+ "class OpenAIAdapter:",
84
+ ' """OpenAI LLM adapter with automatic secret resolution."""',
85
+ "",
86
+ " def __init__(",
87
+ " self,",
88
+ ' api_key: str = secret(env="OPENAI_API_KEY"), # Auto-resolved',
89
+ ' model: str = "gpt-4",',
90
+ " temperature: float = 0.7",
91
+ " ):",
92
+ " self.api_key = api_key",
93
+ " self.model = model",
94
+ " self.temperature = temperature",
95
+ "",
96
+ " async def aresponse(self, messages: list) -> str:",
97
+ " # Your implementation using self.api_key",
98
+ " ...",
99
+ "```",
100
+ "",
101
+ "## Secret Resolution",
102
+ "",
103
+ "Secrets declared with `secret()` are resolved in this order:",
104
+ "1. **Explicit kwargs** - Values passed directly to `__init__`",
105
+ "2. **Environment variables** - From the `env` parameter",
106
+ "3. **Memory port** - From orchestrator memory (with `secret:` prefix)",
107
+ "4. **Error** - If required and no default",
108
+ "",
109
+ "## Available Adapters",
110
+ "",
111
+ ]
112
+
113
+ # Generate adapter tables by port type
114
+ for port_type in sorted(adapters_by_port.keys()):
115
+ port_adapters = adapters_by_port[port_type]
116
+ lines.append(f"### {port_type}")
117
+ lines.append("")
118
+ lines.append("| Adapter | Description |")
119
+ lines.append("|---------|-------------|")
120
+ for adapter in port_adapters:
121
+ desc = adapter.description[:60]
122
+ if len(adapter.description) > 60:
123
+ desc += "..."
124
+ lines.append(f"| `{adapter.name}` | {desc} |")
125
+ lines.append("")
126
+
127
+ # Add YAML usage section
128
+ lines.extend([
129
+ "## Using Adapters in YAML",
130
+ "",
131
+ "```yaml",
132
+ "apiVersion: hexdag/v1",
133
+ "kind: Pipeline",
134
+ "metadata:",
135
+ " name: my-pipeline",
136
+ "spec:",
137
+ " ports:",
138
+ " llm:",
139
+ " adapter: hexdag.builtin.adapters.openai.OpenAIAdapter",
140
+ " config:",
141
+ " api_key: ${OPENAI_API_KEY}",
142
+ " model: gpt-4",
143
+ "",
144
+ " nodes:",
145
+ " - kind: llm_node",
146
+ " metadata:",
147
+ " name: analyzer",
148
+ " spec:",
149
+ ' prompt_template: "Analyze: {{input}}"',
150
+ " dependencies: []",
151
+ "```",
152
+ "",
153
+ "## Best Practices",
154
+ "",
155
+ "1. **Async First**: Use `async def` for I/O operations",
156
+ "2. **Type Hints**: Add type annotations for better tooling",
157
+ "3. **Docstrings**: Document your adapter's purpose and config",
158
+ "4. **Error Handling**: Wrap external calls in try/except",
159
+ "5. **Secrets**: Use `secret()` - never hardcode secrets",
160
+ ])
161
+
162
+ return "\n".join(lines)
163
+
164
+ def generate_node_guide(self, nodes: list[NodeDoc]) -> str:
165
+ """Generate node creation guide.
166
+
167
+ Parameters
168
+ ----------
169
+ nodes : list[NodeDoc]
170
+ List of node documentation objects
171
+
172
+ Returns
173
+ -------
174
+ str
175
+ Complete node guide as markdown
176
+ """
177
+ lines = [
178
+ "# Creating Custom Nodes in hexDAG",
179
+ "",
180
+ "## Overview",
181
+ "",
182
+ "Nodes are the building blocks of hexDAG pipelines. Each node performs a specific",
183
+ "task and can be connected to other nodes via dependencies.",
184
+ "",
185
+ "## Quick Start",
186
+ "",
187
+ "### Using FunctionNode (Simplest)",
188
+ "",
189
+ "Reference any Python function by module path:",
190
+ "",
191
+ "```yaml",
192
+ "- kind: function_node",
193
+ " metadata:",
194
+ " name: my_processor",
195
+ " spec:",
196
+ " fn: mycompany.processors.process_data",
197
+ " dependencies: []",
198
+ "```",
199
+ "",
200
+ "```python",
201
+ "# mycompany/processors.py",
202
+ "def process_data(input_data: dict) -> dict:",
203
+ ' """Your processing logic."""',
204
+ ' return {"result": input_data["value"] * 2}',
205
+ "```",
206
+ "",
207
+ "## Available Node Types",
208
+ "",
209
+ ]
210
+
211
+ # Generate node documentation
212
+ for node in nodes:
213
+ lines.append(f"### {node.name}")
214
+ lines.append("")
215
+ lines.append(f"{node.description}")
216
+ lines.append("")
217
+ lines.append(f"**Kind**: `{node.kind}`")
218
+ lines.append("")
219
+
220
+ if node.parameters:
221
+ lines.append("| Parameter | Type | Required | Description |")
222
+ lines.append("|-----------|------|----------|-------------|")
223
+ for param in node.parameters:
224
+ req = "Yes" if param.required else "No"
225
+ desc = param.description[:50]
226
+ if len(param.description) > 50:
227
+ desc += "..."
228
+ lines.append(f"| `{param.name}` | `{param.type_hint}` | {req} | {desc} |")
229
+ lines.append("")
230
+
231
+ if node.yaml_example:
232
+ lines.append("**Example:**")
233
+ lines.append("```yaml")
234
+ lines.append(node.yaml_example.strip())
235
+ lines.append("```")
236
+ lines.append("")
237
+
238
+ # Add custom node section
239
+ lines.extend([
240
+ "## Creating Custom Nodes",
241
+ "",
242
+ "```python",
243
+ "from hexdag.builtin.nodes import BaseNodeFactory",
244
+ "from hexdag.core.domain.dag import NodeSpec",
245
+ "",
246
+ "class CustomProcessorNode(BaseNodeFactory):",
247
+ ' """Custom node for specialized processing."""',
248
+ "",
249
+ " def __call__(",
250
+ " self,",
251
+ " name: str,",
252
+ " threshold: float = 0.5,",
253
+ " **kwargs",
254
+ " ) -> NodeSpec:",
255
+ " async def process_fn(input_data: dict) -> dict:",
256
+ ' if input_data.get("score", 0) > threshold:',
257
+ ' return {"status": "pass"}',
258
+ ' return {"status": "fail"}',
259
+ "",
260
+ " return NodeSpec(",
261
+ " name=name,",
262
+ " fn=process_fn,",
263
+ ' deps=frozenset(kwargs.get("deps", [])),',
264
+ " )",
265
+ "```",
266
+ "",
267
+ "## Best Practices",
268
+ "",
269
+ "1. **Async Functions**: Use `async def` for the node function",
270
+ "2. **Immutable**: Don't modify input_data; return new dict",
271
+ "3. **Type Hints**: Add types for better IDE support",
272
+ "4. **Docstrings**: Document purpose and parameters",
273
+ ])
274
+
275
+ return "\n".join(lines)
276
+
277
+ def generate_tool_guide(self, tools: list[ToolDoc]) -> str:
278
+ """Generate tool creation guide.
279
+
280
+ Parameters
281
+ ----------
282
+ tools : list[ToolDoc]
283
+ List of tool documentation objects
284
+
285
+ Returns
286
+ -------
287
+ str
288
+ Complete tool guide as markdown
289
+ """
290
+ # Separate sync and async tools
291
+ async_tools = [t for t in tools if t.is_async]
292
+ sync_tools = [t for t in tools if not t.is_async]
293
+
294
+ lines = [
295
+ "# Creating Custom Tools for hexDAG Agents",
296
+ "",
297
+ "## Overview",
298
+ "",
299
+ "Tools are functions that agents can invoke during execution. They enable",
300
+ "agents to interact with external systems, perform calculations, or access data.",
301
+ "",
302
+ "## Quick Start",
303
+ "",
304
+ "```python",
305
+ "def calculate(expression: str) -> str:",
306
+ ' """Evaluate a mathematical expression.',
307
+ "",
308
+ " Args:",
309
+ ' expression: Math expression like "2 + 2"',
310
+ "",
311
+ " Returns:",
312
+ " Result as a string",
313
+ ' """',
314
+ " result = eval(expression) # Use safe evaluation in production",
315
+ " return str(result)",
316
+ "```",
317
+ "",
318
+ "## Built-in Tools",
319
+ "",
320
+ ]
321
+
322
+ # Generate tool documentation
323
+ for tool in tools:
324
+ lines.append(f"### {tool.name}")
325
+ lines.append("")
326
+ lines.append(tool.description)
327
+ lines.append("")
328
+
329
+ if tool.parameters:
330
+ lines.append("**Parameters:**")
331
+ for param in tool.parameters:
332
+ opt = "" if param.required else ", optional"
333
+ default = f" Default: `{param.default}`" if param.default else ""
334
+ lines.append(
335
+ f"- `{param.name}` (`{param.type_hint}`{opt}): {param.description}{default}"
336
+ )
337
+ lines.append("")
338
+
339
+ lines.append(f"**Returns:** `{tool.return_type}`")
340
+ if tool.is_async:
341
+ lines.append("")
342
+ lines.append("*This is an async tool.*")
343
+ lines.append("")
344
+
345
+ # Add usage section
346
+ lines.extend([
347
+ "## Using Tools with Agents",
348
+ "",
349
+ "```yaml",
350
+ "- kind: agent_node",
351
+ " metadata:",
352
+ " name: research_agent",
353
+ " spec:",
354
+ ' initial_prompt_template: "Research: {{topic}}"',
355
+ " max_steps: 5",
356
+ " tools:",
357
+ " - hexdag.builtin.tools.builtin_tools.tool_end",
358
+ " - mycompany.tools.search",
359
+ " dependencies: []",
360
+ "```",
361
+ "",
362
+ "## Tool Invocation Format",
363
+ "",
364
+ "Agents invoke tools using:",
365
+ "```",
366
+ 'INVOKE_TOOL: tool_name(param1="value", param2=123)',
367
+ "```",
368
+ "",
369
+ "## Tool Reference",
370
+ "",
371
+ ])
372
+
373
+ # Add reference tables
374
+ if sync_tools:
375
+ lines.append("### Synchronous Tools")
376
+ lines.append("")
377
+ lines.append("| Tool | Description | Return Type |")
378
+ lines.append("|------|-------------|-------------|")
379
+ for tool in sync_tools:
380
+ desc = tool.description[:40]
381
+ if len(tool.description) > 40:
382
+ desc += "..."
383
+ lines.append(f"| `{tool.name}` | {desc} | `{tool.return_type}` |")
384
+ lines.append("")
385
+
386
+ if async_tools:
387
+ lines.append("### Asynchronous Tools")
388
+ lines.append("")
389
+ lines.append("| Tool | Description | Return Type |")
390
+ lines.append("|------|-------------|-------------|")
391
+ for tool in async_tools:
392
+ desc = tool.description[:40]
393
+ if len(tool.description) > 40:
394
+ desc += "..."
395
+ lines.append(f"| `{tool.name}` | {desc} | `{tool.return_type}` |")
396
+ lines.append("")
397
+
398
+ lines.extend([
399
+ "## Best Practices",
400
+ "",
401
+ "1. **Type Hints**: Always add parameter and return types",
402
+ "2. **Docstrings**: Write clear descriptions for LLM understanding",
403
+ "3. **Error Handling**: Return error messages, don't raise exceptions",
404
+ "4. **Idempotent**: Tools should be safe to retry",
405
+ ])
406
+
407
+ return "\n".join(lines)
408
+
409
+ def generate_syntax_reference(self) -> str:
410
+ """Generate syntax reference guide.
411
+
412
+ Returns
413
+ -------
414
+ str
415
+ Complete syntax reference as markdown
416
+ """
417
+ return """# hexDAG Variable Reference Syntax
418
+
419
+ ## 1. Initial Input Reference: $input
420
+
421
+ Use `$input.field` in `input_mapping` to access the original pipeline input.
422
+
423
+ ```yaml
424
+ nodes:
425
+ - kind: function_node
426
+ metadata:
427
+ name: processor
428
+ spec:
429
+ fn: myapp.process
430
+ input_mapping:
431
+ load_id: $input.load_id
432
+ carrier: $input.carrier_mc
433
+ dependencies: [extractor]
434
+ ```
435
+
436
+ ## 2. Node Output Reference: {{node.field}}
437
+
438
+ Use Jinja2 syntax in prompt templates to reference previous node outputs.
439
+
440
+ ```yaml
441
+ - kind: llm_node
442
+ metadata:
443
+ name: analyzer
444
+ spec:
445
+ prompt_template: |
446
+ Analyze this data:
447
+ {{extractor.result}}
448
+ ```
449
+
450
+ ## 3. Environment Variables: ${VAR}
451
+
452
+ ```yaml
453
+ spec:
454
+ ports:
455
+ llm:
456
+ config:
457
+ model: ${MODEL} # Resolved at build time
458
+ api_key: ${OPENAI_API_KEY} # Secret - resolved at runtime
459
+ ```
460
+
461
+ **Secret Patterns (deferred to runtime):**
462
+ - `*_API_KEY`, `*_SECRET`, `*_TOKEN`, `*_PASSWORD`, `*_CREDENTIAL`, `SECRET_*`
463
+
464
+ ## 4. Input Mapping
465
+
466
+ ```yaml
467
+ - kind: function_node
468
+ metadata:
469
+ name: merger
470
+ spec:
471
+ fn: myapp.merge_results
472
+ input_mapping:
473
+ request_id: $input.id # From initial input
474
+ analysis: analyzer.result # From dependency
475
+ dependencies: [analyzer]
476
+ ```
477
+
478
+ ## Quick Reference
479
+
480
+ | Syntax | Location | Purpose |
481
+ |--------|----------|---------|
482
+ | `$input.field` | input_mapping | Access initial pipeline input |
483
+ | `{{node.field}}` | prompt_template | Jinja2 template reference |
484
+ | `${VAR}` | Any string | Environment variable |
485
+ | `${VAR:default}` | Any string | Env var with default |
486
+ | `node.path` | input_mapping | Dependency output extraction |
487
+ """
488
+
489
+ def generate_extension_guide(
490
+ self,
491
+ adapters: list[AdapterDoc],
492
+ nodes: list[NodeDoc],
493
+ tools: list[ToolDoc],
494
+ ) -> str:
495
+ """Generate extension overview guide.
496
+
497
+ Parameters
498
+ ----------
499
+ adapters : list[AdapterDoc]
500
+ List of adapter documentation objects
501
+ nodes : list[NodeDoc]
502
+ List of node documentation objects
503
+ tools : list[ToolDoc]
504
+ List of tool documentation objects
505
+
506
+ Returns
507
+ -------
508
+ str
509
+ Complete extension guide as markdown
510
+ """
511
+ lines = [
512
+ "# Extending hexDAG - Overview",
513
+ "",
514
+ "## Extension Points",
515
+ "",
516
+ "| Component | Purpose | Available |",
517
+ "|-----------|---------|-----------|",
518
+ f"| **Adapter** | Connect to external services | {len(adapters)} |",
519
+ f"| **Node** | Custom processing logic | {len(nodes)} |",
520
+ f"| **Tool** | Agent-callable functions | {len(tools)} |",
521
+ "",
522
+ "## Quick Reference",
523
+ "",
524
+ "### Adapters",
525
+ "Use `get_custom_adapter_guide()` for full documentation.",
526
+ "",
527
+ "### Nodes",
528
+ "Use `get_custom_node_guide()` for full documentation.",
529
+ "",
530
+ "### Tools",
531
+ "Use `get_custom_tool_guide()` for full documentation.",
532
+ "",
533
+ "## MCP Tools for Development",
534
+ "",
535
+ "| Tool | Purpose |",
536
+ "|------|---------|",
537
+ "| `list_nodes()` | See available nodes |",
538
+ "| `list_adapters()` | See available adapters |",
539
+ "| `list_tools()` | See available tools |",
540
+ "| `get_component_schema()` | Get config schema |",
541
+ "| `validate_yaml_pipeline()` | Validate your YAML |",
542
+ "| `get_pipeline_schema()` | Get full JSON schema |",
543
+ ]
544
+
545
+ return "\n".join(lines)
546
+
547
+ def generate_pipeline_schema_guide(self) -> str:
548
+ """Generate pipeline schema reference guide from JSON schema.
549
+
550
+ Reads the auto-generated pipeline-schema.json and produces
551
+ a human-readable markdown reference.
552
+
553
+ Returns
554
+ -------
555
+ str
556
+ Complete pipeline schema guide as markdown
557
+ """
558
+ lines = [
559
+ "# hexDAG Pipeline Schema Reference",
560
+ "",
561
+ "This reference is auto-generated from the pipeline JSON schema.",
562
+ "",
563
+ "## Overview",
564
+ "",
565
+ "hexDAG pipelines are defined in YAML using a Kubernetes-like structure.",
566
+ "The schema provides validation and IDE autocompletion support.",
567
+ "",
568
+ "## Pipeline Structure",
569
+ "",
570
+ "```yaml",
571
+ "apiVersion: hexdag/v1",
572
+ "kind: Pipeline",
573
+ "metadata:",
574
+ " name: my-pipeline",
575
+ " description: Pipeline description",
576
+ "spec:",
577
+ " ports: {} # Adapter configurations",
578
+ " nodes: [] # Processing nodes",
579
+ " events: {} # Event handlers",
580
+ "```",
581
+ "",
582
+ ]
583
+
584
+ # Try to load and parse schema
585
+ try:
586
+ schema = self._load_schema()
587
+ if schema:
588
+ lines.extend(self._generate_node_types_section(schema))
589
+ lines.extend(self._generate_ports_section(schema))
590
+ lines.extend(self._generate_events_section(schema))
591
+ except Exception as e:
592
+ logger.warning(f"Could not load pipeline schema: {e}")
593
+ lines.extend([
594
+ "## Node Types",
595
+ "",
596
+ "*Schema not available. Run `scripts/generate_schemas.py` first.*",
597
+ "",
598
+ ])
599
+
600
+ # Add IDE setup section
601
+ lines.extend([
602
+ "## IDE Setup",
603
+ "",
604
+ "### VS Code",
605
+ "",
606
+ "Add to `.vscode/settings.json`:",
607
+ "",
608
+ "```json",
609
+ "{",
610
+ ' "yaml.schemas": {',
611
+ ' "./schemas/pipeline-schema.json": ["*.yaml", "pipelines/*.yaml"]',
612
+ " }",
613
+ "}",
614
+ "```",
615
+ "",
616
+ "### Schema Location",
617
+ "",
618
+ (
619
+ "The schema file is at `schemas/pipeline-schema.json` and is "
620
+ "auto-generated from node `_yaml_schema` attributes."
621
+ ),
622
+ ])
623
+
624
+ return "\n".join(lines)
625
+
626
+ def _load_schema(self) -> dict[str, Any] | None:
627
+ """Load the pipeline schema JSON file.
628
+
629
+ Returns
630
+ -------
631
+ dict[str, Any] | None
632
+ Parsed schema or None if not found
633
+ """
634
+ if not SCHEMA_PATH.exists():
635
+ return None
636
+ return json.loads(SCHEMA_PATH.read_text())
637
+
638
+ def _generate_node_types_section(self, schema: dict[str, Any]) -> list[str]:
639
+ """Generate node types documentation from schema.
640
+
641
+ Parameters
642
+ ----------
643
+ schema : dict[str, Any]
644
+ Parsed JSON schema
645
+
646
+ Returns
647
+ -------
648
+ list[str]
649
+ Lines of markdown documentation
650
+ """
651
+ lines = ["## Node Types", ""]
652
+
653
+ defs = schema.get("$defs", {})
654
+
655
+ # Find all node specs
656
+ node_specs = [(name, spec) for name, spec in defs.items() if name.endswith("NodeSpec")]
657
+
658
+ if not node_specs:
659
+ lines.append("*No node types found in schema.*")
660
+ lines.append("")
661
+ return lines
662
+
663
+ # Generate table
664
+ lines.append("| Node Kind | Description |")
665
+ lines.append("|-----------|-------------|")
666
+
667
+ for name, spec in sorted(node_specs):
668
+ kind = name.replace("NodeSpec", "").lower()
669
+ # Convert CamelCase to snake_case
670
+ kind = "".join(f"_{c.lower()}" if c.isupper() else c for c in kind).lstrip("_")
671
+
672
+ # Get description from spec
673
+ desc = spec.get("description", "")
674
+ if not desc:
675
+ props = spec.get("properties", {})
676
+ spec_prop = props.get("spec", {})
677
+ desc = spec_prop.get("description", "No description")
678
+
679
+ # Truncate long descriptions
680
+ if len(desc) > 60:
681
+ desc = desc[:57] + "..."
682
+
683
+ lines.append(f"| `{kind}_node` | {desc} |")
684
+
685
+ lines.append("")
686
+
687
+ # Generate detailed sections for each node
688
+ for name, spec in sorted(node_specs):
689
+ lines.extend(self._generate_node_detail(name, spec))
690
+
691
+ return lines
692
+
693
+ def _generate_node_detail(self, name: str, spec: dict[str, Any]) -> list[str]:
694
+ """Generate detailed documentation for a single node type.
695
+
696
+ Parameters
697
+ ----------
698
+ name : str
699
+ Node spec name (e.g., "FunctionNodeSpec")
700
+ spec : dict[str, Any]
701
+ Node specification from schema
702
+
703
+ Returns
704
+ -------
705
+ list[str]
706
+ Lines of markdown documentation
707
+ """
708
+ kind = name.replace("NodeSpec", "").lower()
709
+ kind = "".join(f"_{c.lower()}" if c.isupper() else c for c in kind).lstrip("_")
710
+
711
+ lines = [f"### {kind}_node", ""]
712
+
713
+ # Get description
714
+ props = spec.get("properties", {})
715
+ spec_prop = props.get("spec", {})
716
+ desc = spec_prop.get("description", spec.get("description", ""))
717
+ if desc:
718
+ lines.append(desc)
719
+ lines.append("")
720
+
721
+ # Extract spec properties
722
+ spec_props = spec_prop.get("properties", {})
723
+ required = spec_prop.get("required", [])
724
+
725
+ if spec_props:
726
+ lines.append("**Parameters:**")
727
+ lines.append("")
728
+ lines.append("| Parameter | Type | Required | Description |")
729
+ lines.append("|-----------|------|----------|-------------|")
730
+
731
+ for param_name, param_spec in sorted(spec_props.items()):
732
+ param_type = self._get_type_from_schema(param_spec)
733
+ is_required = "Yes" if param_name in required else "No"
734
+ param_desc = param_spec.get("description", "")
735
+ if len(param_desc) > 40:
736
+ param_desc = param_desc[:37] + "..."
737
+ lines.append(f"| `{param_name}` | {param_type} | {is_required} | {param_desc} |")
738
+
739
+ lines.append("")
740
+
741
+ # Add example
742
+ lines.append("**Example:**")
743
+ lines.append("")
744
+ lines.append("```yaml")
745
+ lines.append(f"- kind: {kind}_node")
746
+ lines.append(" metadata:")
747
+ lines.append(f" name: my_{kind}")
748
+ lines.append(" spec:")
749
+
750
+ # Add required params as example (limit to first 3)
751
+ lines.extend(f" {param_name}: # required" for param_name in required[:3])
752
+
753
+ lines.append(" dependencies: []")
754
+ lines.append("```")
755
+ lines.append("")
756
+
757
+ return lines
758
+
759
+ def _get_type_from_schema(self, spec: dict[str, Any]) -> str:
760
+ """Extract type string from JSON schema property.
761
+
762
+ Parameters
763
+ ----------
764
+ spec : dict[str, Any]
765
+ Property specification
766
+
767
+ Returns
768
+ -------
769
+ str
770
+ Human-readable type string
771
+ """
772
+ if "const" in spec:
773
+ return f'`"{spec["const"]}"`'
774
+
775
+ if "enum" in spec:
776
+ return " | ".join(f'`"{v}`"' for v in spec["enum"][:3])
777
+
778
+ if "anyOf" in spec:
779
+ types = [self._get_type_from_schema(s) for s in spec["anyOf"][:2]]
780
+ return " | ".join(types)
781
+
782
+ type_val = spec.get("type")
783
+ if isinstance(type_val, list):
784
+ return " | ".join(type_val)
785
+ if type_val == "array":
786
+ items = spec.get("items", {})
787
+ item_type = items.get("type", "any")
788
+ return f"list[{item_type}]"
789
+ if type_val:
790
+ return type_val
791
+
792
+ return "any"
793
+
794
+ def _generate_ports_section(self, schema: dict[str, Any]) -> list[str]:
795
+ """Generate ports documentation from schema.
796
+
797
+ Parameters
798
+ ----------
799
+ schema : dict[str, Any]
800
+ Parsed JSON schema
801
+
802
+ Returns
803
+ -------
804
+ list[str]
805
+ Lines of markdown documentation
806
+ """
807
+ return [
808
+ "## Ports Configuration",
809
+ "",
810
+ "Ports connect pipelines to external services:",
811
+ "",
812
+ "```yaml",
813
+ "spec:",
814
+ " ports:",
815
+ " llm:",
816
+ " adapter: hexdag.builtin.adapters.openai.OpenAIAdapter",
817
+ " config:",
818
+ " api_key: ${OPENAI_API_KEY}",
819
+ " model: gpt-4",
820
+ " memory:",
821
+ " adapter: hexdag.builtin.adapters.memory.InMemoryMemory",
822
+ " database:",
823
+ " adapter: hexdag.builtin.adapters.database.sqlite.SQLiteAdapter",
824
+ " config:",
825
+ " db_path: ./data.db",
826
+ "```",
827
+ "",
828
+ "### Available Port Types",
829
+ "",
830
+ "| Port | Purpose |",
831
+ "|------|---------|",
832
+ "| `llm` | Language model interactions |",
833
+ "| `memory` | Persistent agent memory |",
834
+ "| `database` | Data persistence |",
835
+ "| `secret` | Secret/credential management |",
836
+ "| `tool_router` | Tool invocation routing |",
837
+ "",
838
+ ]
839
+
840
+ def _generate_events_section(self, schema: dict[str, Any]) -> list[str]:
841
+ """Generate events documentation from schema.
842
+
843
+ Parameters
844
+ ----------
845
+ schema : dict[str, Any]
846
+ Parsed JSON schema
847
+
848
+ Returns
849
+ -------
850
+ list[str]
851
+ Lines of markdown documentation
852
+ """
853
+ return [
854
+ "## Events Configuration",
855
+ "",
856
+ "Configure event handlers for observability:",
857
+ "",
858
+ "```yaml",
859
+ "spec:",
860
+ " events:",
861
+ " node_failed:",
862
+ " - type: alert",
863
+ " target: pagerduty",
864
+ " severity: high",
865
+ " pipeline_completed:",
866
+ " - type: metrics",
867
+ " target: datadog",
868
+ "```",
869
+ "",
870
+ "### Event Types",
871
+ "",
872
+ "| Event | When Triggered |",
873
+ "|-------|----------------|",
874
+ "| `pipeline_started` | Pipeline execution begins |",
875
+ "| `pipeline_completed` | Pipeline execution finishes |",
876
+ "| `node_started` | Node execution begins |",
877
+ "| `node_completed` | Node execution finishes |",
878
+ "| `node_failed` | Node execution fails |",
879
+ "",
880
+ "### Handler Types",
881
+ "",
882
+ "| Type | Purpose |",
883
+ "|------|---------|",
884
+ "| `alert` | Send alerts (PagerDuty, Slack) |",
885
+ "| `metrics` | Emit metrics (Datadog, Prometheus) |",
886
+ "| `log` | Write to logs |",
887
+ "| `webhook` | Call external webhooks |",
888
+ "| `callback` | Execute Python callbacks |",
889
+ "",
890
+ ]