kubiya-control-plane-api 0.9.15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (479) hide show
  1. control_plane_api/LICENSE +676 -0
  2. control_plane_api/README.md +350 -0
  3. control_plane_api/__init__.py +4 -0
  4. control_plane_api/__version__.py +8 -0
  5. control_plane_api/alembic/README +1 -0
  6. control_plane_api/alembic/env.py +121 -0
  7. control_plane_api/alembic/script.py.mako +28 -0
  8. control_plane_api/alembic/versions/2613c65c3dbe_initial_database_setup.py +32 -0
  9. control_plane_api/alembic/versions/2df520d4927d_merge_heads.py +28 -0
  10. control_plane_api/alembic/versions/43abf98d6a01_add_paused_status_to_executions.py +73 -0
  11. control_plane_api/alembic/versions/6289854264cb_merge_multiple_heads.py +28 -0
  12. control_plane_api/alembic/versions/6a4d4dc3d8dc_generate_execution_transitions.py +50 -0
  13. control_plane_api/alembic/versions/87d11cf0a783_add_disconnected_status_to_worker_.py +44 -0
  14. control_plane_api/alembic/versions/add_ephemeral_queue_support.py +85 -0
  15. control_plane_api/alembic/versions/add_model_type_to_llm_models.py +31 -0
  16. control_plane_api/alembic/versions/add_plan_executions_table.py +114 -0
  17. control_plane_api/alembic/versions/add_trace_span_tables.py +154 -0
  18. control_plane_api/alembic/versions/add_user_info_to_traces.py +36 -0
  19. control_plane_api/alembic/versions/adjusting_foreign_keys.py +32 -0
  20. control_plane_api/alembic/versions/b4983d976db2_initial_tables.py +1128 -0
  21. control_plane_api/alembic/versions/d181a3b40e71_rename_custom_metadata_to_metadata_in_.py +50 -0
  22. control_plane_api/alembic/versions/df9117888e82_add_missing_columns.py +82 -0
  23. control_plane_api/alembic/versions/f25de6ad895a_missing_migrations.py +34 -0
  24. control_plane_api/alembic/versions/f71305fb69b9_fix_ephemeral_queue_deletion_foreign_key.py +54 -0
  25. control_plane_api/alembic/versions/mark_local_exec_queues_as_ephemeral.py +68 -0
  26. control_plane_api/alembic.ini +148 -0
  27. control_plane_api/api/index.py +12 -0
  28. control_plane_api/app/__init__.py +11 -0
  29. control_plane_api/app/activities/__init__.py +20 -0
  30. control_plane_api/app/activities/agent_activities.py +384 -0
  31. control_plane_api/app/activities/plan_generation_activities.py +499 -0
  32. control_plane_api/app/activities/team_activities.py +424 -0
  33. control_plane_api/app/activities/temporal_cloud_activities.py +588 -0
  34. control_plane_api/app/config/__init__.py +35 -0
  35. control_plane_api/app/config/api_config.py +469 -0
  36. control_plane_api/app/config/config_loader.py +224 -0
  37. control_plane_api/app/config/model_pricing.py +323 -0
  38. control_plane_api/app/config/storage_config.py +159 -0
  39. control_plane_api/app/config.py +115 -0
  40. control_plane_api/app/controllers/__init__.py +0 -0
  41. control_plane_api/app/controllers/execution_environment_controller.py +1315 -0
  42. control_plane_api/app/database.py +135 -0
  43. control_plane_api/app/exceptions.py +408 -0
  44. control_plane_api/app/lib/__init__.py +11 -0
  45. control_plane_api/app/lib/environment.py +65 -0
  46. control_plane_api/app/lib/event_bus/__init__.py +17 -0
  47. control_plane_api/app/lib/event_bus/base.py +136 -0
  48. control_plane_api/app/lib/event_bus/manager.py +335 -0
  49. control_plane_api/app/lib/event_bus/providers/__init__.py +6 -0
  50. control_plane_api/app/lib/event_bus/providers/http_provider.py +166 -0
  51. control_plane_api/app/lib/event_bus/providers/nats_provider.py +324 -0
  52. control_plane_api/app/lib/event_bus/providers/redis_provider.py +233 -0
  53. control_plane_api/app/lib/event_bus/providers/websocket_provider.py +497 -0
  54. control_plane_api/app/lib/job_executor.py +330 -0
  55. control_plane_api/app/lib/kubiya_client.py +293 -0
  56. control_plane_api/app/lib/litellm_pricing.py +166 -0
  57. control_plane_api/app/lib/mcp_validation.py +163 -0
  58. control_plane_api/app/lib/nats/__init__.py +13 -0
  59. control_plane_api/app/lib/nats/credentials_manager.py +288 -0
  60. control_plane_api/app/lib/nats/listener.py +374 -0
  61. control_plane_api/app/lib/planning_prompt_builder.py +153 -0
  62. control_plane_api/app/lib/planning_tools/__init__.py +41 -0
  63. control_plane_api/app/lib/planning_tools/agents.py +409 -0
  64. control_plane_api/app/lib/planning_tools/agno_toolkit.py +836 -0
  65. control_plane_api/app/lib/planning_tools/base.py +119 -0
  66. control_plane_api/app/lib/planning_tools/cognitive_memory_tools.py +403 -0
  67. control_plane_api/app/lib/planning_tools/context_graph_tools.py +545 -0
  68. control_plane_api/app/lib/planning_tools/environments.py +218 -0
  69. control_plane_api/app/lib/planning_tools/knowledge.py +204 -0
  70. control_plane_api/app/lib/planning_tools/models.py +93 -0
  71. control_plane_api/app/lib/planning_tools/planning_service.py +646 -0
  72. control_plane_api/app/lib/planning_tools/resources.py +242 -0
  73. control_plane_api/app/lib/planning_tools/teams.py +334 -0
  74. control_plane_api/app/lib/policy_enforcer_client.py +1016 -0
  75. control_plane_api/app/lib/redis_client.py +803 -0
  76. control_plane_api/app/lib/sqlalchemy_utils.py +486 -0
  77. control_plane_api/app/lib/state_transition_tools/__init__.py +7 -0
  78. control_plane_api/app/lib/state_transition_tools/execution_context.py +388 -0
  79. control_plane_api/app/lib/storage/__init__.py +20 -0
  80. control_plane_api/app/lib/storage/base_provider.py +274 -0
  81. control_plane_api/app/lib/storage/provider_factory.py +157 -0
  82. control_plane_api/app/lib/storage/vercel_blob_provider.py +468 -0
  83. control_plane_api/app/lib/supabase.py +71 -0
  84. control_plane_api/app/lib/supabase_utils.py +138 -0
  85. control_plane_api/app/lib/task_planning/__init__.py +138 -0
  86. control_plane_api/app/lib/task_planning/agent_factory.py +308 -0
  87. control_plane_api/app/lib/task_planning/agents.py +389 -0
  88. control_plane_api/app/lib/task_planning/cache.py +218 -0
  89. control_plane_api/app/lib/task_planning/entity_resolver.py +273 -0
  90. control_plane_api/app/lib/task_planning/helpers.py +293 -0
  91. control_plane_api/app/lib/task_planning/hooks.py +474 -0
  92. control_plane_api/app/lib/task_planning/models.py +503 -0
  93. control_plane_api/app/lib/task_planning/plan_validator.py +166 -0
  94. control_plane_api/app/lib/task_planning/planning_workflow.py +2911 -0
  95. control_plane_api/app/lib/task_planning/runner.py +656 -0
  96. control_plane_api/app/lib/task_planning/streaming_hook.py +213 -0
  97. control_plane_api/app/lib/task_planning/workflow.py +424 -0
  98. control_plane_api/app/lib/templating/__init__.py +88 -0
  99. control_plane_api/app/lib/templating/compiler.py +278 -0
  100. control_plane_api/app/lib/templating/engine.py +178 -0
  101. control_plane_api/app/lib/templating/parsers/__init__.py +29 -0
  102. control_plane_api/app/lib/templating/parsers/base.py +96 -0
  103. control_plane_api/app/lib/templating/parsers/env.py +85 -0
  104. control_plane_api/app/lib/templating/parsers/graph.py +112 -0
  105. control_plane_api/app/lib/templating/parsers/secret.py +87 -0
  106. control_plane_api/app/lib/templating/parsers/simple.py +81 -0
  107. control_plane_api/app/lib/templating/resolver.py +366 -0
  108. control_plane_api/app/lib/templating/types.py +214 -0
  109. control_plane_api/app/lib/templating/validator.py +201 -0
  110. control_plane_api/app/lib/temporal_client.py +232 -0
  111. control_plane_api/app/lib/temporal_credentials_cache.py +178 -0
  112. control_plane_api/app/lib/temporal_credentials_service.py +203 -0
  113. control_plane_api/app/lib/validation/__init__.py +24 -0
  114. control_plane_api/app/lib/validation/runtime_validation.py +388 -0
  115. control_plane_api/app/main.py +531 -0
  116. control_plane_api/app/middleware/__init__.py +10 -0
  117. control_plane_api/app/middleware/auth.py +645 -0
  118. control_plane_api/app/middleware/exception_handler.py +267 -0
  119. control_plane_api/app/middleware/prometheus_middleware.py +173 -0
  120. control_plane_api/app/middleware/rate_limiting.py +384 -0
  121. control_plane_api/app/middleware/request_id.py +202 -0
  122. control_plane_api/app/models/__init__.py +40 -0
  123. control_plane_api/app/models/agent.py +90 -0
  124. control_plane_api/app/models/analytics.py +206 -0
  125. control_plane_api/app/models/associations.py +107 -0
  126. control_plane_api/app/models/auth_user.py +73 -0
  127. control_plane_api/app/models/context.py +161 -0
  128. control_plane_api/app/models/custom_integration.py +99 -0
  129. control_plane_api/app/models/environment.py +64 -0
  130. control_plane_api/app/models/execution.py +125 -0
  131. control_plane_api/app/models/execution_transition.py +50 -0
  132. control_plane_api/app/models/job.py +159 -0
  133. control_plane_api/app/models/llm_model.py +78 -0
  134. control_plane_api/app/models/orchestration.py +66 -0
  135. control_plane_api/app/models/plan_execution.py +102 -0
  136. control_plane_api/app/models/presence.py +49 -0
  137. control_plane_api/app/models/project.py +61 -0
  138. control_plane_api/app/models/project_management.py +85 -0
  139. control_plane_api/app/models/session.py +29 -0
  140. control_plane_api/app/models/skill.py +155 -0
  141. control_plane_api/app/models/system_tables.py +43 -0
  142. control_plane_api/app/models/task_planning.py +372 -0
  143. control_plane_api/app/models/team.py +86 -0
  144. control_plane_api/app/models/trace.py +257 -0
  145. control_plane_api/app/models/user_profile.py +54 -0
  146. control_plane_api/app/models/worker.py +221 -0
  147. control_plane_api/app/models/workflow.py +161 -0
  148. control_plane_api/app/models/workspace.py +50 -0
  149. control_plane_api/app/observability/__init__.py +177 -0
  150. control_plane_api/app/observability/context_logging.py +475 -0
  151. control_plane_api/app/observability/decorators.py +337 -0
  152. control_plane_api/app/observability/local_span_processor.py +702 -0
  153. control_plane_api/app/observability/metrics.py +303 -0
  154. control_plane_api/app/observability/middleware.py +246 -0
  155. control_plane_api/app/observability/optional.py +115 -0
  156. control_plane_api/app/observability/tracing.py +382 -0
  157. control_plane_api/app/policies/README.md +149 -0
  158. control_plane_api/app/policies/approved_users.rego +62 -0
  159. control_plane_api/app/policies/business_hours.rego +51 -0
  160. control_plane_api/app/policies/rate_limiting.rego +100 -0
  161. control_plane_api/app/policies/tool_enforcement/README.md +336 -0
  162. control_plane_api/app/policies/tool_enforcement/bash_command_validation.rego +71 -0
  163. control_plane_api/app/policies/tool_enforcement/business_hours_enforcement.rego +82 -0
  164. control_plane_api/app/policies/tool_enforcement/mcp_tool_allowlist.rego +58 -0
  165. control_plane_api/app/policies/tool_enforcement/production_safeguards.rego +80 -0
  166. control_plane_api/app/policies/tool_enforcement/role_based_tool_access.rego +44 -0
  167. control_plane_api/app/policies/tool_restrictions.rego +86 -0
  168. control_plane_api/app/routers/__init__.py +4 -0
  169. control_plane_api/app/routers/agents.py +382 -0
  170. control_plane_api/app/routers/agents_v2.py +1598 -0
  171. control_plane_api/app/routers/analytics.py +1310 -0
  172. control_plane_api/app/routers/auth.py +59 -0
  173. control_plane_api/app/routers/client_config.py +57 -0
  174. control_plane_api/app/routers/context_graph.py +561 -0
  175. control_plane_api/app/routers/context_manager.py +577 -0
  176. control_plane_api/app/routers/custom_integrations.py +490 -0
  177. control_plane_api/app/routers/enforcer.py +132 -0
  178. control_plane_api/app/routers/environment_context.py +252 -0
  179. control_plane_api/app/routers/environments.py +761 -0
  180. control_plane_api/app/routers/execution_environment.py +847 -0
  181. control_plane_api/app/routers/executions/__init__.py +28 -0
  182. control_plane_api/app/routers/executions/router.py +286 -0
  183. control_plane_api/app/routers/executions/services/__init__.py +22 -0
  184. control_plane_api/app/routers/executions/services/demo_worker_health.py +156 -0
  185. control_plane_api/app/routers/executions/services/status_service.py +420 -0
  186. control_plane_api/app/routers/executions/services/test_worker_health.py +480 -0
  187. control_plane_api/app/routers/executions/services/worker_health.py +514 -0
  188. control_plane_api/app/routers/executions/streaming/__init__.py +22 -0
  189. control_plane_api/app/routers/executions/streaming/deduplication.py +352 -0
  190. control_plane_api/app/routers/executions/streaming/event_buffer.py +353 -0
  191. control_plane_api/app/routers/executions/streaming/event_formatter.py +964 -0
  192. control_plane_api/app/routers/executions/streaming/history_loader.py +588 -0
  193. control_plane_api/app/routers/executions/streaming/live_source.py +693 -0
  194. control_plane_api/app/routers/executions/streaming/streamer.py +849 -0
  195. control_plane_api/app/routers/executions.py +4888 -0
  196. control_plane_api/app/routers/health.py +165 -0
  197. control_plane_api/app/routers/health_v2.py +394 -0
  198. control_plane_api/app/routers/integration_templates.py +496 -0
  199. control_plane_api/app/routers/integrations.py +287 -0
  200. control_plane_api/app/routers/jobs.py +1809 -0
  201. control_plane_api/app/routers/metrics.py +517 -0
  202. control_plane_api/app/routers/models.py +82 -0
  203. control_plane_api/app/routers/models_v2.py +628 -0
  204. control_plane_api/app/routers/plan_executions.py +1481 -0
  205. control_plane_api/app/routers/plan_generation_async.py +304 -0
  206. control_plane_api/app/routers/policies.py +669 -0
  207. control_plane_api/app/routers/presence.py +234 -0
  208. control_plane_api/app/routers/projects.py +987 -0
  209. control_plane_api/app/routers/runners.py +379 -0
  210. control_plane_api/app/routers/runtimes.py +172 -0
  211. control_plane_api/app/routers/secrets.py +171 -0
  212. control_plane_api/app/routers/skills.py +1010 -0
  213. control_plane_api/app/routers/skills_definitions.py +140 -0
  214. control_plane_api/app/routers/storage.py +456 -0
  215. control_plane_api/app/routers/task_planning.py +611 -0
  216. control_plane_api/app/routers/task_queues.py +650 -0
  217. control_plane_api/app/routers/team_context.py +274 -0
  218. control_plane_api/app/routers/teams.py +1747 -0
  219. control_plane_api/app/routers/templates.py +248 -0
  220. control_plane_api/app/routers/traces.py +571 -0
  221. control_plane_api/app/routers/websocket_client.py +479 -0
  222. control_plane_api/app/routers/websocket_executions_status.py +437 -0
  223. control_plane_api/app/routers/websocket_gateway.py +323 -0
  224. control_plane_api/app/routers/websocket_traces.py +576 -0
  225. control_plane_api/app/routers/worker_queues.py +2555 -0
  226. control_plane_api/app/routers/worker_websocket.py +419 -0
  227. control_plane_api/app/routers/workers.py +1004 -0
  228. control_plane_api/app/routers/workflows.py +204 -0
  229. control_plane_api/app/runtimes/__init__.py +6 -0
  230. control_plane_api/app/runtimes/validation.py +344 -0
  231. control_plane_api/app/schemas/__init__.py +1 -0
  232. control_plane_api/app/schemas/job_schemas.py +302 -0
  233. control_plane_api/app/schemas/mcp_schemas.py +311 -0
  234. control_plane_api/app/schemas/template_schemas.py +133 -0
  235. control_plane_api/app/schemas/trace_schemas.py +168 -0
  236. control_plane_api/app/schemas/worker_queue_observability_schemas.py +165 -0
  237. control_plane_api/app/services/__init__.py +1 -0
  238. control_plane_api/app/services/agno_planning_strategy.py +233 -0
  239. control_plane_api/app/services/agno_service.py +838 -0
  240. control_plane_api/app/services/claude_code_planning_service.py +203 -0
  241. control_plane_api/app/services/context_graph_client.py +224 -0
  242. control_plane_api/app/services/custom_integration_service.py +415 -0
  243. control_plane_api/app/services/integration_resolution_service.py +345 -0
  244. control_plane_api/app/services/litellm_service.py +394 -0
  245. control_plane_api/app/services/plan_generator.py +79 -0
  246. control_plane_api/app/services/planning_strategy.py +66 -0
  247. control_plane_api/app/services/planning_strategy_factory.py +118 -0
  248. control_plane_api/app/services/policy_service.py +615 -0
  249. control_plane_api/app/services/state_transition_service.py +755 -0
  250. control_plane_api/app/services/storage_service.py +593 -0
  251. control_plane_api/app/services/temporal_cloud_provisioning.py +150 -0
  252. control_plane_api/app/services/toolsets/context_graph_skill.py +432 -0
  253. control_plane_api/app/services/trace_retention.py +354 -0
  254. control_plane_api/app/services/worker_queue_metrics_service.py +190 -0
  255. control_plane_api/app/services/workflow_cancellation_manager.py +135 -0
  256. control_plane_api/app/services/workflow_operations_service.py +611 -0
  257. control_plane_api/app/skills/__init__.py +100 -0
  258. control_plane_api/app/skills/base.py +239 -0
  259. control_plane_api/app/skills/builtin/__init__.py +37 -0
  260. control_plane_api/app/skills/builtin/agent_communication/__init__.py +8 -0
  261. control_plane_api/app/skills/builtin/agent_communication/skill.py +246 -0
  262. control_plane_api/app/skills/builtin/code_ingestion/__init__.py +4 -0
  263. control_plane_api/app/skills/builtin/code_ingestion/skill.py +267 -0
  264. control_plane_api/app/skills/builtin/cognitive_memory/__init__.py +4 -0
  265. control_plane_api/app/skills/builtin/cognitive_memory/skill.py +174 -0
  266. control_plane_api/app/skills/builtin/contextual_awareness/__init__.py +4 -0
  267. control_plane_api/app/skills/builtin/contextual_awareness/skill.py +387 -0
  268. control_plane_api/app/skills/builtin/data_visualization/__init__.py +4 -0
  269. control_plane_api/app/skills/builtin/data_visualization/skill.py +154 -0
  270. control_plane_api/app/skills/builtin/docker/__init__.py +4 -0
  271. control_plane_api/app/skills/builtin/docker/skill.py +104 -0
  272. control_plane_api/app/skills/builtin/file_generation/__init__.py +4 -0
  273. control_plane_api/app/skills/builtin/file_generation/skill.py +94 -0
  274. control_plane_api/app/skills/builtin/file_system/__init__.py +4 -0
  275. control_plane_api/app/skills/builtin/file_system/skill.py +110 -0
  276. control_plane_api/app/skills/builtin/knowledge_api/__init__.py +5 -0
  277. control_plane_api/app/skills/builtin/knowledge_api/skill.py +124 -0
  278. control_plane_api/app/skills/builtin/python/__init__.py +4 -0
  279. control_plane_api/app/skills/builtin/python/skill.py +92 -0
  280. control_plane_api/app/skills/builtin/remote_filesystem/__init__.py +5 -0
  281. control_plane_api/app/skills/builtin/remote_filesystem/skill.py +170 -0
  282. control_plane_api/app/skills/builtin/shell/__init__.py +4 -0
  283. control_plane_api/app/skills/builtin/shell/skill.py +161 -0
  284. control_plane_api/app/skills/builtin/slack/__init__.py +3 -0
  285. control_plane_api/app/skills/builtin/slack/skill.py +302 -0
  286. control_plane_api/app/skills/builtin/workflow_executor/__init__.py +4 -0
  287. control_plane_api/app/skills/builtin/workflow_executor/skill.py +469 -0
  288. control_plane_api/app/skills/business_intelligence.py +189 -0
  289. control_plane_api/app/skills/config.py +63 -0
  290. control_plane_api/app/skills/loaders/__init__.py +14 -0
  291. control_plane_api/app/skills/loaders/base.py +73 -0
  292. control_plane_api/app/skills/loaders/filesystem_loader.py +199 -0
  293. control_plane_api/app/skills/registry.py +125 -0
  294. control_plane_api/app/utils/helpers.py +12 -0
  295. control_plane_api/app/utils/workflow_executor.py +354 -0
  296. control_plane_api/app/workflows/__init__.py +11 -0
  297. control_plane_api/app/workflows/agent_execution.py +520 -0
  298. control_plane_api/app/workflows/agent_execution_with_skills.py +223 -0
  299. control_plane_api/app/workflows/namespace_provisioning.py +326 -0
  300. control_plane_api/app/workflows/plan_generation.py +254 -0
  301. control_plane_api/app/workflows/team_execution.py +442 -0
  302. control_plane_api/scripts/seed_models.py +240 -0
  303. control_plane_api/scripts/validate_existing_tool_names.py +492 -0
  304. control_plane_api/shared/__init__.py +8 -0
  305. control_plane_api/shared/version.py +17 -0
  306. control_plane_api/test_deduplication.py +274 -0
  307. control_plane_api/test_executor_deduplication_e2e.py +309 -0
  308. control_plane_api/test_job_execution_e2e.py +283 -0
  309. control_plane_api/test_real_integration.py +193 -0
  310. control_plane_api/version.py +38 -0
  311. control_plane_api/worker/__init__.py +0 -0
  312. control_plane_api/worker/activities/__init__.py +0 -0
  313. control_plane_api/worker/activities/agent_activities.py +1585 -0
  314. control_plane_api/worker/activities/approval_activities.py +234 -0
  315. control_plane_api/worker/activities/job_activities.py +199 -0
  316. control_plane_api/worker/activities/runtime_activities.py +1167 -0
  317. control_plane_api/worker/activities/skill_activities.py +282 -0
  318. control_plane_api/worker/activities/team_activities.py +479 -0
  319. control_plane_api/worker/agent_runtime_server.py +370 -0
  320. control_plane_api/worker/binary_manager.py +333 -0
  321. control_plane_api/worker/config/__init__.py +31 -0
  322. control_plane_api/worker/config/worker_config.py +273 -0
  323. control_plane_api/worker/control_plane_client.py +1491 -0
  324. control_plane_api/worker/examples/analytics_integration_example.py +362 -0
  325. control_plane_api/worker/health_monitor.py +159 -0
  326. control_plane_api/worker/metrics.py +237 -0
  327. control_plane_api/worker/models/__init__.py +1 -0
  328. control_plane_api/worker/models/error_events.py +105 -0
  329. control_plane_api/worker/models/inputs.py +89 -0
  330. control_plane_api/worker/runtimes/__init__.py +35 -0
  331. control_plane_api/worker/runtimes/agent_runtime/runtime.py +485 -0
  332. control_plane_api/worker/runtimes/agno/__init__.py +34 -0
  333. control_plane_api/worker/runtimes/agno/config.py +248 -0
  334. control_plane_api/worker/runtimes/agno/hooks.py +385 -0
  335. control_plane_api/worker/runtimes/agno/mcp_builder.py +195 -0
  336. control_plane_api/worker/runtimes/agno/runtime.py +1063 -0
  337. control_plane_api/worker/runtimes/agno/utils.py +163 -0
  338. control_plane_api/worker/runtimes/base.py +979 -0
  339. control_plane_api/worker/runtimes/claude_code/__init__.py +38 -0
  340. control_plane_api/worker/runtimes/claude_code/cleanup.py +184 -0
  341. control_plane_api/worker/runtimes/claude_code/client_pool.py +529 -0
  342. control_plane_api/worker/runtimes/claude_code/config.py +829 -0
  343. control_plane_api/worker/runtimes/claude_code/hooks.py +482 -0
  344. control_plane_api/worker/runtimes/claude_code/litellm_proxy.py +1702 -0
  345. control_plane_api/worker/runtimes/claude_code/mcp_builder.py +467 -0
  346. control_plane_api/worker/runtimes/claude_code/mcp_discovery.py +558 -0
  347. control_plane_api/worker/runtimes/claude_code/runtime.py +1546 -0
  348. control_plane_api/worker/runtimes/claude_code/tool_mapper.py +403 -0
  349. control_plane_api/worker/runtimes/claude_code/utils.py +149 -0
  350. control_plane_api/worker/runtimes/factory.py +173 -0
  351. control_plane_api/worker/runtimes/model_utils.py +107 -0
  352. control_plane_api/worker/runtimes/validation.py +93 -0
  353. control_plane_api/worker/services/__init__.py +1 -0
  354. control_plane_api/worker/services/agent_communication_tools.py +908 -0
  355. control_plane_api/worker/services/agent_executor.py +485 -0
  356. control_plane_api/worker/services/agent_executor_v2.py +793 -0
  357. control_plane_api/worker/services/analytics_collector.py +457 -0
  358. control_plane_api/worker/services/analytics_service.py +464 -0
  359. control_plane_api/worker/services/approval_tools.py +310 -0
  360. control_plane_api/worker/services/approval_tools_agno.py +207 -0
  361. control_plane_api/worker/services/cancellation_manager.py +177 -0
  362. control_plane_api/worker/services/code_ingestion_tools.py +465 -0
  363. control_plane_api/worker/services/contextual_awareness_tools.py +405 -0
  364. control_plane_api/worker/services/data_visualization.py +834 -0
  365. control_plane_api/worker/services/event_publisher.py +531 -0
  366. control_plane_api/worker/services/jira_tools.py +257 -0
  367. control_plane_api/worker/services/remote_filesystem_tools.py +498 -0
  368. control_plane_api/worker/services/runtime_analytics.py +328 -0
  369. control_plane_api/worker/services/session_service.py +365 -0
  370. control_plane_api/worker/services/skill_context_enhancement.py +181 -0
  371. control_plane_api/worker/services/skill_factory.py +471 -0
  372. control_plane_api/worker/services/system_prompt_enhancement.py +410 -0
  373. control_plane_api/worker/services/team_executor.py +715 -0
  374. control_plane_api/worker/services/team_executor_v2.py +1866 -0
  375. control_plane_api/worker/services/tool_enforcement.py +254 -0
  376. control_plane_api/worker/services/workflow_executor/__init__.py +52 -0
  377. control_plane_api/worker/services/workflow_executor/event_processor.py +287 -0
  378. control_plane_api/worker/services/workflow_executor/event_publisher.py +210 -0
  379. control_plane_api/worker/services/workflow_executor/executors/__init__.py +15 -0
  380. control_plane_api/worker/services/workflow_executor/executors/base.py +270 -0
  381. control_plane_api/worker/services/workflow_executor/executors/json_executor.py +50 -0
  382. control_plane_api/worker/services/workflow_executor/executors/python_executor.py +50 -0
  383. control_plane_api/worker/services/workflow_executor/models.py +142 -0
  384. control_plane_api/worker/services/workflow_executor_tools.py +1748 -0
  385. control_plane_api/worker/skills/__init__.py +12 -0
  386. control_plane_api/worker/skills/builtin/context_graph_search/README.md +213 -0
  387. control_plane_api/worker/skills/builtin/context_graph_search/__init__.py +5 -0
  388. control_plane_api/worker/skills/builtin/context_graph_search/agno_impl.py +808 -0
  389. control_plane_api/worker/skills/builtin/context_graph_search/skill.yaml +67 -0
  390. control_plane_api/worker/skills/builtin/contextual_awareness/__init__.py +4 -0
  391. control_plane_api/worker/skills/builtin/contextual_awareness/agno_impl.py +62 -0
  392. control_plane_api/worker/skills/builtin/data_visualization/agno_impl.py +18 -0
  393. control_plane_api/worker/skills/builtin/data_visualization/skill.yaml +84 -0
  394. control_plane_api/worker/skills/builtin/docker/agno_impl.py +65 -0
  395. control_plane_api/worker/skills/builtin/docker/skill.yaml +60 -0
  396. control_plane_api/worker/skills/builtin/file_generation/agno_impl.py +47 -0
  397. control_plane_api/worker/skills/builtin/file_generation/skill.yaml +64 -0
  398. control_plane_api/worker/skills/builtin/file_system/agno_impl.py +32 -0
  399. control_plane_api/worker/skills/builtin/file_system/skill.yaml +54 -0
  400. control_plane_api/worker/skills/builtin/knowledge_api/__init__.py +4 -0
  401. control_plane_api/worker/skills/builtin/knowledge_api/agno_impl.py +50 -0
  402. control_plane_api/worker/skills/builtin/knowledge_api/skill.yaml +66 -0
  403. control_plane_api/worker/skills/builtin/python/agno_impl.py +25 -0
  404. control_plane_api/worker/skills/builtin/python/skill.yaml +60 -0
  405. control_plane_api/worker/skills/builtin/schema_fix_mixin.py +260 -0
  406. control_plane_api/worker/skills/builtin/shell/agno_impl.py +31 -0
  407. control_plane_api/worker/skills/builtin/shell/skill.yaml +60 -0
  408. control_plane_api/worker/skills/builtin/slack/__init__.py +3 -0
  409. control_plane_api/worker/skills/builtin/slack/agno_impl.py +1282 -0
  410. control_plane_api/worker/skills/builtin/slack/skill.yaml +276 -0
  411. control_plane_api/worker/skills/builtin/workflow_executor/agno_impl.py +62 -0
  412. control_plane_api/worker/skills/builtin/workflow_executor/skill.yaml +79 -0
  413. control_plane_api/worker/skills/loaders/__init__.py +5 -0
  414. control_plane_api/worker/skills/loaders/base.py +23 -0
  415. control_plane_api/worker/skills/loaders/filesystem_loader.py +357 -0
  416. control_plane_api/worker/skills/registry.py +208 -0
  417. control_plane_api/worker/tests/__init__.py +1 -0
  418. control_plane_api/worker/tests/conftest.py +12 -0
  419. control_plane_api/worker/tests/e2e/__init__.py +0 -0
  420. control_plane_api/worker/tests/e2e/test_context_graph_real_api.py +338 -0
  421. control_plane_api/worker/tests/e2e/test_context_graph_templates_e2e.py +523 -0
  422. control_plane_api/worker/tests/e2e/test_enforcement_e2e.py +344 -0
  423. control_plane_api/worker/tests/e2e/test_execution_flow.py +571 -0
  424. control_plane_api/worker/tests/e2e/test_single_execution_mode.py +656 -0
  425. control_plane_api/worker/tests/integration/__init__.py +0 -0
  426. control_plane_api/worker/tests/integration/test_builtin_skills_fixes.py +245 -0
  427. control_plane_api/worker/tests/integration/test_context_graph_search_integration.py +365 -0
  428. control_plane_api/worker/tests/integration/test_control_plane_integration.py +308 -0
  429. control_plane_api/worker/tests/integration/test_hook_enforcement_integration.py +579 -0
  430. control_plane_api/worker/tests/integration/test_scheduled_job_workflow.py +237 -0
  431. control_plane_api/worker/tests/integration/test_system_prompt_enhancement_integration.py +343 -0
  432. control_plane_api/worker/tests/unit/__init__.py +0 -0
  433. control_plane_api/worker/tests/unit/test_builtin_skill_autoload.py +396 -0
  434. control_plane_api/worker/tests/unit/test_context_graph_search.py +450 -0
  435. control_plane_api/worker/tests/unit/test_context_graph_templates.py +403 -0
  436. control_plane_api/worker/tests/unit/test_control_plane_client.py +401 -0
  437. control_plane_api/worker/tests/unit/test_control_plane_client_jobs.py +345 -0
  438. control_plane_api/worker/tests/unit/test_job_activities.py +353 -0
  439. control_plane_api/worker/tests/unit/test_skill_context_enhancement.py +321 -0
  440. control_plane_api/worker/tests/unit/test_system_prompt_enhancement.py +415 -0
  441. control_plane_api/worker/tests/unit/test_tool_enforcement.py +324 -0
  442. control_plane_api/worker/utils/__init__.py +1 -0
  443. control_plane_api/worker/utils/chunk_batcher.py +330 -0
  444. control_plane_api/worker/utils/environment.py +65 -0
  445. control_plane_api/worker/utils/error_publisher.py +260 -0
  446. control_plane_api/worker/utils/event_batcher.py +256 -0
  447. control_plane_api/worker/utils/logging_config.py +335 -0
  448. control_plane_api/worker/utils/logging_helper.py +326 -0
  449. control_plane_api/worker/utils/parameter_validator.py +120 -0
  450. control_plane_api/worker/utils/retry_utils.py +60 -0
  451. control_plane_api/worker/utils/streaming_utils.py +665 -0
  452. control_plane_api/worker/utils/tool_validation.py +332 -0
  453. control_plane_api/worker/utils/workspace_manager.py +163 -0
  454. control_plane_api/worker/websocket_client.py +393 -0
  455. control_plane_api/worker/worker.py +1297 -0
  456. control_plane_api/worker/workflows/__init__.py +0 -0
  457. control_plane_api/worker/workflows/agent_execution.py +909 -0
  458. control_plane_api/worker/workflows/scheduled_job_wrapper.py +332 -0
  459. control_plane_api/worker/workflows/team_execution.py +611 -0
  460. kubiya_control_plane_api-0.9.15.dist-info/METADATA +354 -0
  461. kubiya_control_plane_api-0.9.15.dist-info/RECORD +479 -0
  462. kubiya_control_plane_api-0.9.15.dist-info/WHEEL +5 -0
  463. kubiya_control_plane_api-0.9.15.dist-info/entry_points.txt +5 -0
  464. kubiya_control_plane_api-0.9.15.dist-info/licenses/LICENSE +676 -0
  465. kubiya_control_plane_api-0.9.15.dist-info/top_level.txt +3 -0
  466. scripts/__init__.py +1 -0
  467. scripts/migrations.py +39 -0
  468. scripts/seed_worker_queues.py +128 -0
  469. scripts/setup_agent_runtime.py +142 -0
  470. worker_internal/__init__.py +1 -0
  471. worker_internal/planner/__init__.py +1 -0
  472. worker_internal/planner/activities.py +1499 -0
  473. worker_internal/planner/agent_tools.py +197 -0
  474. worker_internal/planner/event_models.py +148 -0
  475. worker_internal/planner/event_publisher.py +67 -0
  476. worker_internal/planner/models.py +199 -0
  477. worker_internal/planner/retry_logic.py +134 -0
  478. worker_internal/planner/worker.py +300 -0
  479. worker_internal/planner/workflows.py +970 -0
@@ -0,0 +1,1546 @@
1
+ """
2
+ Claude Code runtime implementation using Claude Code SDK.
3
+
4
+ This runtime adapter integrates the Claude Code SDK to power agents with
5
+ advanced coding capabilities, file operations, and specialized tools.
6
+
7
+ ALL 7 BUGS FIXED:
8
+ - Bug #1: Added metadata = {} initialization
9
+ - Bug #2: Replaced print() with logger.debug()
10
+ - Bug #3: Made MCP fallback patterns explicit
11
+ - Bug #4: Added session_id validation
12
+ - Bug #5: Added explicit disconnect() calls with timeout
13
+ - Bug #6: Added tool name validation
14
+ - Bug #7: Removed debug output
15
+ """
16
+
17
+ from typing import Dict, Any, Optional, AsyncIterator, Callable, TYPE_CHECKING
18
+ import structlog
19
+ import asyncio
20
+ import time
21
+ import os
22
+ from temporalio import activity
23
+
24
+ from ..base import (
25
+ RuntimeType,
26
+ RuntimeExecutionResult,
27
+ RuntimeExecutionContext,
28
+ RuntimeCapabilities,
29
+ BaseRuntime,
30
+ RuntimeRegistry,
31
+ )
32
+ from .config import build_claude_options
33
+ from .utils import (
34
+ extract_usage_from_result_message,
35
+ extract_session_id_from_result_message,
36
+ build_prompt_with_history,
37
+ )
38
+ from .litellm_proxy import clear_execution_context
39
+ from .cleanup import cleanup_sdk_client
40
+ from .client_pool import ClaudeCodeClientPool
41
+
42
+ if TYPE_CHECKING:
43
+ from control_plane_client import ControlPlaneClient
44
+ from services.cancellation_manager import CancellationManager
45
+
46
+ logger = structlog.get_logger(__name__)
47
+
48
+ # ⚡ PERFORMANCE: Lazy load Claude SDK at module level (not per-execution)
49
+ # This imports the SDK once when the module loads, making subsequent executions faster
50
+ _CLAUDE_SDK_AVAILABLE = False
51
+ _CLAUDE_SDK_IMPORT_ERROR = None
52
+ _SDK_CLASSES = {}
53
+
54
+ try:
55
+ from claude_agent_sdk import (
56
+ ClaudeSDKClient,
57
+ AssistantMessage,
58
+ ResultMessage,
59
+ TextBlock,
60
+ ToolUseBlock,
61
+ ToolResultBlock,
62
+ )
63
+ _CLAUDE_SDK_AVAILABLE = True
64
+ _SDK_CLASSES = {
65
+ 'ClaudeSDKClient': ClaudeSDKClient,
66
+ 'AssistantMessage': AssistantMessage,
67
+ 'ResultMessage': ResultMessage,
68
+ 'TextBlock': TextBlock,
69
+ 'ToolUseBlock': ToolUseBlock,
70
+ 'ToolResultBlock': ToolResultBlock,
71
+ }
72
+ logger.info("claude_code_sdk_preloaded", status="success")
73
+ except ImportError as e:
74
+ _CLAUDE_SDK_IMPORT_ERROR = str(e)
75
+ logger.warning("claude_code_sdk_not_available", error=str(e))
76
+
77
+
78
+ @RuntimeRegistry.register(RuntimeType.CLAUDE_CODE)
79
+ class ClaudeCodeRuntime(BaseRuntime):
80
+ """
81
+ Runtime implementation using Claude Code SDK.
82
+
83
+ This runtime leverages Claude Code's specialized capabilities for
84
+ software engineering tasks, file operations, and developer workflows.
85
+
86
+ Features:
87
+ - Streaming execution with real-time updates
88
+ - Conversation history support via ClaudeSDKClient
89
+ - Custom tool integration via MCP
90
+ - Hooks for tool execution monitoring
91
+ - Cancellation support via interrupt()
92
+
93
+ All critical bugs have been fixed in this refactored version.
94
+ """
95
+
96
+ def __init__(
97
+ self,
98
+ control_plane_client: "ControlPlaneClient",
99
+ cancellation_manager: "CancellationManager",
100
+ **kwargs,
101
+ ):
102
+ """
103
+ Initialize the Claude Code runtime.
104
+
105
+ Args:
106
+ control_plane_client: Client for Control Plane API
107
+ cancellation_manager: Manager for execution cancellation
108
+ **kwargs: Additional configuration options
109
+ """
110
+ super().__init__(control_plane_client, cancellation_manager, **kwargs)
111
+
112
+ # Track active SDK clients for cancellation
113
+ self._active_clients: Dict[str, Any] = {}
114
+
115
+ # Track custom MCP servers
116
+ self._custom_mcp_servers: Dict[str, Any] = {} # server_name -> mcp_server
117
+
118
+ # Cache MCP discovery results (discovered once, reused per execution)
119
+ # Format: {server_name: {tools: [...], resources: [...], prompts: [...], connected: bool}}
120
+ self._mcp_discovery_cache: Dict[str, Any] = {}
121
+ self._mcp_cache_lock = None # Will be initialized on first use
122
+
123
+ def get_runtime_type(self) -> RuntimeType:
124
+ """Return RuntimeType.CLAUDE_CODE."""
125
+ return RuntimeType.CLAUDE_CODE
126
+
127
+ def get_capabilities(self) -> RuntimeCapabilities:
128
+ """Return Claude Code runtime capabilities."""
129
+ return RuntimeCapabilities(
130
+ streaming=True,
131
+ tools=True,
132
+ mcp=True,
133
+ hooks=True,
134
+ cancellation=True,
135
+ conversation_history=True,
136
+ custom_tools=True,
137
+ )
138
+
139
+ async def _execute_impl(
140
+ self, context: RuntimeExecutionContext
141
+ ) -> RuntimeExecutionResult:
142
+ """
143
+ Execute agent using Claude Code SDK (non-streaming).
144
+
145
+ Production-grade implementation with:
146
+ - Comprehensive error handling
147
+ - Proper resource cleanup
148
+ - Detailed logging
149
+ - Timeout management
150
+ - Graceful degradation
151
+
152
+ BUG FIX #1: Added metadata = {} initialization
153
+ BUG FIX #5: Added explicit disconnect() call
154
+
155
+ Args:
156
+ context: Execution context with prompt, history, config
157
+
158
+ Returns:
159
+ RuntimeExecutionResult with response and metadata
160
+ """
161
+ client = None
162
+ start_time = asyncio.get_event_loop().time()
163
+
164
+ try:
165
+ # ⚡ PERFORMANCE: Use pre-loaded SDK classes (loaded at module level)
166
+ if not _CLAUDE_SDK_AVAILABLE:
167
+ return RuntimeExecutionResult(
168
+ response="",
169
+ usage={},
170
+ success=False,
171
+ error=f"Claude Code SDK not available: {_CLAUDE_SDK_IMPORT_ERROR}",
172
+ )
173
+
174
+ ClaudeSDKClient = _SDK_CLASSES['ClaudeSDKClient']
175
+ ResultMessage = _SDK_CLASSES['ResultMessage']
176
+
177
+ self.logger.info(
178
+ "starting_claude_code_non_streaming_execution",
179
+ execution_id=context.execution_id,
180
+ model=context.model_id,
181
+ has_history=bool(context.conversation_history),
182
+ )
183
+
184
+ # Build options and create client
185
+ options, _, _, _ = await build_claude_options(context, runtime=self)
186
+
187
+ # Merge custom MCP servers
188
+ if self._custom_mcp_servers:
189
+ if not options.mcp_servers:
190
+ options.mcp_servers = {}
191
+
192
+ for server_name, mcp_server in self._custom_mcp_servers.items():
193
+ options.mcp_servers[server_name] = mcp_server
194
+
195
+ # Add tool names to allowed_tools for permission
196
+ if hasattr(mcp_server, 'tools') and mcp_server.tools:
197
+ for tool in mcp_server.tools:
198
+ if hasattr(tool, 'name'):
199
+ tool_name = f"mcp__{server_name}__{tool.name}"
200
+ if tool_name not in options.allowed_tools:
201
+ options.allowed_tools.append(tool_name)
202
+
203
+ self.logger.debug(
204
+ "custom_mcp_server_added_non_streaming",
205
+ server_name=server_name,
206
+ execution_id=context.execution_id
207
+ )
208
+
209
+ # Suppress verbose MCP STDIO parsing error logs
210
+ # These errors occur when MCP servers incorrectly log to stdout instead of stderr
211
+ # The connection continues to work, but the SDK logs errors for each non-JSONRPC line
212
+ import logging
213
+ mcp_stdio_logger = logging.getLogger("mcp.client.stdio")
214
+ original_stdio_level = mcp_stdio_logger.level
215
+ mcp_stdio_logger.setLevel(logging.ERROR) # Only show critical errors
216
+
217
+ # Create and connect client
218
+ client = ClaudeSDKClient(options=options)
219
+
220
+ try:
221
+ await client.connect()
222
+ except Exception as e:
223
+ error_msg = str(e)
224
+
225
+ # Build comprehensive error message from all sources
226
+ # The SDK wraps subprocess errors, so "No conversation found" may be in:
227
+ # 1. The exception message itself
228
+ # 2. The stderr attribute (if CalledProcessError)
229
+ # 3. The output attribute
230
+ # 4. The __cause__ chain
231
+ full_error_context = error_msg
232
+ if hasattr(e, 'stderr') and e.stderr:
233
+ stderr_content = e.stderr if isinstance(e.stderr, str) else e.stderr.decode('utf-8', errors='replace')
234
+ full_error_context += f" | stderr: {stderr_content}"
235
+ if hasattr(e, 'output') and e.output:
236
+ output_content = e.output if isinstance(e.output, str) else e.output.decode('utf-8', errors='replace')
237
+ full_error_context += f" | output: {output_content}"
238
+ if e.__cause__:
239
+ full_error_context += f" | cause: {str(e.__cause__)}"
240
+
241
+ # Detect session-related errors:
242
+ # 1. Explicit "No conversation found" in any error context
243
+ # 2. "exit code 1" when we were trying to resume (likely session issue)
244
+ is_session_error = (
245
+ "No conversation found" in full_error_context or
246
+ "conversation" in full_error_context.lower() or
247
+ # Fallback: exit code 1 during resume attempt is likely a session issue
248
+ (options.resume and "exit code 1" in error_msg.lower())
249
+ )
250
+
251
+ # Handle session-related errors gracefully by retrying without resume
252
+ if is_session_error and options.resume:
253
+ logger.warning(
254
+ "session_resume_failed_retrying_without_resume",
255
+ execution_id=context.execution_id[:16],
256
+ error=error_msg,
257
+ full_error_context=full_error_context[:500], # Truncate for logging
258
+ session_id=options.resume[:16] if options.resume else None,
259
+ note="Session doesn't exist or is invalid, creating new conversation"
260
+ )
261
+
262
+ # Retry without resume parameter
263
+ options.resume = None
264
+ session_resume_failed = True # Track for prompt building
265
+ client = ClaudeSDKClient(options=options)
266
+ await client.connect()
267
+ logger.info(
268
+ "client_connected_without_resume",
269
+ execution_id=context.execution_id[:16]
270
+ )
271
+ else:
272
+ # Different error, re-raise
273
+ raise
274
+ else:
275
+ session_resume_failed = False
276
+
277
+ self._active_clients[context.execution_id] = client
278
+
279
+ # Send prompt - include history in prompt if session resume failed
280
+ # (since we can't rely on SDK's session continuity)
281
+ if session_resume_failed and context.conversation_history:
282
+ prompt = build_prompt_with_history(context)
283
+ logger.info(
284
+ "using_prompt_with_history_fallback",
285
+ execution_id=context.execution_id[:16],
286
+ history_messages=len(context.conversation_history),
287
+ note="Session resume failed, including history in prompt"
288
+ )
289
+ else:
290
+ prompt = context.prompt
291
+
292
+ self.logger.debug(
293
+ "sending_query_to_claude_code_sdk",
294
+ execution_id=context.execution_id,
295
+ prompt_length=len(prompt),
296
+ using_session_resume=bool(options.resume),
297
+ )
298
+
299
+ await client.query(prompt)
300
+
301
+ # Collect complete response
302
+ response_text = ""
303
+ usage = {}
304
+ tool_messages = []
305
+ finish_reason = None
306
+ message_count = 0
307
+ last_heartbeat = asyncio.get_event_loop().time() # Track last heartbeat for Temporal activity liveness
308
+
309
+ # BUG FIX #1: Initialize metadata before use
310
+ metadata = {}
311
+
312
+ # Use receive_response() to get messages until ResultMessage
313
+ async for message in client.receive_response():
314
+ message_count += 1
315
+
316
+ # Send heartbeat every 5 seconds or every 10 messages
317
+ current_time = asyncio.get_event_loop().time()
318
+ if current_time - last_heartbeat > 5 or message_count % 10 == 0:
319
+ try:
320
+ activity.heartbeat({
321
+ "status": "processing",
322
+ "messages_received": message_count,
323
+ "response_length": len(response_text),
324
+ "elapsed_seconds": int(current_time - last_heartbeat)
325
+ })
326
+ last_heartbeat = current_time
327
+ except Exception as e:
328
+ # Non-fatal: heartbeat failure should not break execution
329
+ self.logger.warning("heartbeat_failed_non_fatal", execution_id=context.execution_id, error=str(e))
330
+
331
+ # Extract content from AssistantMessage
332
+ if hasattr(message, "content"):
333
+ for block in message.content:
334
+ if hasattr(block, "text"):
335
+ response_text += block.text
336
+ elif hasattr(block, "name"): # ToolUseBlock
337
+ tool_messages.append(
338
+ {
339
+ "tool": block.name,
340
+ "input": getattr(block, "input", {}),
341
+ "tool_use_id": getattr(block, "id", None),
342
+ }
343
+ )
344
+
345
+ # Extract usage, finish reason, and session_id from ResultMessage
346
+ if isinstance(message, ResultMessage):
347
+ usage = extract_usage_from_result_message(message)
348
+
349
+ if usage:
350
+ self.logger.info(
351
+ "claude_code_usage_extracted",
352
+ execution_id=context.execution_id[:8],
353
+ input_tokens=usage["input_tokens"],
354
+ output_tokens=usage["output_tokens"],
355
+ cache_read=usage["cache_read_tokens"],
356
+ )
357
+
358
+ finish_reason = message.subtype # "success" or "error"
359
+
360
+ # BUG FIX #4: Extract and validate session_id
361
+ session_id = extract_session_id_from_result_message(
362
+ message, context.execution_id
363
+ )
364
+
365
+ if session_id:
366
+ # BUG FIX #1: metadata is now properly initialized
367
+ metadata["claude_code_session_id"] = session_id
368
+
369
+ self.logger.info(
370
+ "claude_code_execution_completed",
371
+ execution_id=context.execution_id[:8],
372
+ finish_reason=finish_reason,
373
+ message_count=message_count,
374
+ response_length=len(response_text),
375
+ tool_count=len(tool_messages),
376
+ tokens=usage.get("total_tokens", 0),
377
+ has_session_id=bool(session_id),
378
+ )
379
+ break
380
+
381
+ elapsed_time = asyncio.get_event_loop().time() - start_time
382
+
383
+ # Merge metadata with execution stats
384
+ final_metadata = {
385
+ **metadata, # Includes claude_code_session_id if present
386
+ "elapsed_time": elapsed_time,
387
+ "message_count": message_count,
388
+ }
389
+
390
+ return RuntimeExecutionResult(
391
+ response=response_text,
392
+ usage=usage,
393
+ success=finish_reason == "success",
394
+ finish_reason=finish_reason or "stop",
395
+ tool_execution_messages=tool_messages, # Use standard field name for analytics
396
+ tool_messages=tool_messages, # Keep for backward compatibility
397
+ model=context.model_id,
398
+ metadata=final_metadata,
399
+ )
400
+
401
+ except ImportError as e:
402
+ self.logger.error(
403
+ "claude_code_sdk_not_installed",
404
+ execution_id=context.execution_id,
405
+ error=str(e),
406
+ )
407
+
408
+ # Publish error event
409
+ try:
410
+ from control_plane_api.worker.utils.error_publisher import (
411
+ ErrorEventPublisher, ErrorSeverity, ErrorCategory
412
+ )
413
+ error_publisher = ErrorEventPublisher(self.control_plane)
414
+ await error_publisher.publish_error(
415
+ execution_id=context.execution_id,
416
+ exception=e,
417
+ severity=ErrorSeverity.CRITICAL,
418
+ category=ErrorCategory.RUNTIME_INIT,
419
+ stage="initialization",
420
+ component="claude_code_runtime",
421
+ operation="sdk_import",
422
+ recovery_actions=[
423
+ "Install Claude Code SDK: pip install claude-agent-sdk",
424
+ "Verify SDK version is compatible",
425
+ "Check Python environment configuration",
426
+ ],
427
+ )
428
+ except Exception as publish_error:
429
+ # Log error publishing failure but don't let it break execution flow
430
+ self.logger.error(
431
+ "error_publish_failed",
432
+ error=str(publish_error),
433
+ error_type=type(publish_error).__name__,
434
+ original_error="Claude Code SDK not available",
435
+ execution_id=context.execution_id
436
+ )
437
+
438
+ return RuntimeExecutionResult(
439
+ response="",
440
+ usage={},
441
+ success=False,
442
+ error=f"Claude Code SDK not available: {str(e)}",
443
+ )
444
+
445
+ except asyncio.TimeoutError:
446
+ elapsed_time = asyncio.get_event_loop().time() - start_time
447
+ self.logger.error(
448
+ "claude_code_execution_timeout",
449
+ execution_id=context.execution_id,
450
+ elapsed_time=elapsed_time,
451
+ )
452
+
453
+ # Publish timeout error event
454
+ try:
455
+ from control_plane_api.worker.utils.error_publisher import (
456
+ ErrorEventPublisher, ErrorSeverity, ErrorCategory
457
+ )
458
+ error_publisher = ErrorEventPublisher(self.control_plane)
459
+ await error_publisher.publish_error(
460
+ execution_id=context.execution_id,
461
+ exception=asyncio.TimeoutError("Execution timeout exceeded"),
462
+ severity=ErrorSeverity.ERROR,
463
+ category=ErrorCategory.TIMEOUT,
464
+ stage="execution",
465
+ component="claude_code_runtime",
466
+ operation="agent_execution",
467
+ metadata={"elapsed_time": elapsed_time},
468
+ recovery_actions=[
469
+ "Simplify the prompt or reduce complexity",
470
+ "Increase timeout settings if appropriate",
471
+ "Check system resources and load",
472
+ ],
473
+ )
474
+ except Exception as publish_error:
475
+ # Log error publishing failure but don't let it break execution flow
476
+ self.logger.error(
477
+ "error_publish_failed",
478
+ error=str(publish_error),
479
+ error_type=type(publish_error).__name__,
480
+ original_error="Execution timeout",
481
+ execution_id=context.execution_id
482
+ )
483
+
484
+ return RuntimeExecutionResult(
485
+ response="",
486
+ usage={},
487
+ success=False,
488
+ error="Execution timeout exceeded",
489
+ )
490
+
491
+ except asyncio.CancelledError:
492
+ self.logger.warning(
493
+ "claude_code_execution_cancelled_gracefully",
494
+ execution_id=context.execution_id,
495
+ )
496
+ # DURABILITY FIX: Do NOT re-raise! Handle cancellation gracefully
497
+ # Return partial result to allow workflow to handle interruption
498
+ return RuntimeExecutionResult(
499
+ response="", # No response accumulated in non-streaming mode
500
+ usage={},
501
+ success=False, # Non-streaming cancellation is a failure (no partial state)
502
+ error="Execution was cancelled",
503
+ finish_reason="cancelled",
504
+ metadata={"interrupted": True, "can_resume": False},
505
+ )
506
+
507
+ except Exception as e:
508
+ self.logger.error(
509
+ "claude_code_execution_failed",
510
+ execution_id=context.execution_id,
511
+ error=str(e),
512
+ error_type=type(e).__name__,
513
+ exc_info=True,
514
+ )
515
+
516
+ # Publish generic error event with stack trace
517
+ try:
518
+ from control_plane_api.worker.utils.error_publisher import (
519
+ ErrorEventPublisher, ErrorSeverity, ErrorCategory
520
+ )
521
+ error_publisher = ErrorEventPublisher(self.control_plane)
522
+ await error_publisher.publish_error(
523
+ execution_id=context.execution_id,
524
+ exception=e,
525
+ severity=ErrorSeverity.CRITICAL,
526
+ category=ErrorCategory.UNKNOWN,
527
+ stage="execution",
528
+ component="claude_code_runtime",
529
+ operation="agent_execution",
530
+ include_stack_trace=True,
531
+ )
532
+ except Exception as publish_error:
533
+ # Log error publishing failure but don't let it break execution flow
534
+ self.logger.error(
535
+ "error_publish_failed",
536
+ error=str(publish_error),
537
+ error_type=type(publish_error).__name__,
538
+ original_error=f"{type(e).__name__}: {str(e)}",
539
+ execution_id=context.execution_id
540
+ )
541
+
542
+ return RuntimeExecutionResult(
543
+ response="",
544
+ usage={},
545
+ success=False,
546
+ error=f"{type(e).__name__}: {str(e)}",
547
+ )
548
+
549
+ finally:
550
+ # Clear execution context from proxy (with delay to allow in-flight SDK requests)
551
+ try:
552
+ clear_execution_context(
553
+ context.execution_id,
554
+ immediate=False, # Use delayed cleanup
555
+ delay_seconds=5.0 # Wait for in-flight SDK requests
556
+ )
557
+ except Exception as e:
558
+ self.logger.warning(
559
+ "failed_to_clear_proxy_context",
560
+ execution_id=context.execution_id,
561
+ error=str(e),
562
+ )
563
+
564
+ # Restore MCP STDIO log level
565
+ try:
566
+ import logging
567
+ mcp_stdio_logger = logging.getLogger("mcp.client.stdio")
568
+ if 'original_stdio_level' in locals():
569
+ mcp_stdio_logger.setLevel(original_stdio_level)
570
+ except Exception as log_level_error:
571
+ # Log but ignore errors restoring log level - this is non-critical cleanup
572
+ self.logger.debug(
573
+ "failed_to_restore_log_level",
574
+ error=str(log_level_error),
575
+ execution_id=context.execution_id
576
+ )
577
+
578
+ # CRITICAL: Cleanup SDK client
579
+ if context.execution_id in self._active_clients:
580
+ client = self._active_clients.pop(context.execution_id)
581
+ cleanup_sdk_client(client, context.execution_id, self.logger)
582
+
583
+ async def _stream_execute_impl(
584
+ self,
585
+ context: RuntimeExecutionContext,
586
+ event_callback: Optional[Callable[[Dict], None]] = None,
587
+ ) -> AsyncIterator[RuntimeExecutionResult]:
588
+ """
589
+ Production-grade streaming execution with Claude Code SDK.
590
+
591
+ This implementation provides:
592
+ - Comprehensive error handling with specific exception types
593
+ - Detailed structured logging at each stage
594
+ - Proper resource cleanup with finally blocks
595
+ - Real-time event callbacks for tool execution
596
+ - Accumulated metrics and metadata tracking
597
+
598
+ BUG FIX #5: Added explicit disconnect() call
599
+ BUG FIX #7: Removed all debug output
600
+
601
+ Args:
602
+ context: Execution context with prompt, history, config
603
+ event_callback: Optional callback for real-time events
604
+
605
+ Yields:
606
+ RuntimeExecutionResult chunks as they arrive, ending with final metadata
607
+ """
608
+ client = None
609
+ start_time = asyncio.get_event_loop().time()
610
+ chunk_count = 0
611
+
612
+ try:
613
+ # ⚡ PERFORMANCE: Use pre-loaded SDK classes (loaded at module level)
614
+ if not _CLAUDE_SDK_AVAILABLE:
615
+ yield RuntimeExecutionResult(
616
+ response="",
617
+ usage={},
618
+ success=False,
619
+ error=f"Claude Code SDK not available: {_CLAUDE_SDK_IMPORT_ERROR}",
620
+ finish_reason="error",
621
+ tool_messages=[],
622
+ tool_execution_messages=[],
623
+ )
624
+ return
625
+
626
+ ClaudeSDKClient = _SDK_CLASSES['ClaudeSDKClient']
627
+ AssistantMessage = _SDK_CLASSES['AssistantMessage']
628
+ ResultMessage = _SDK_CLASSES['ResultMessage']
629
+ TextBlock = _SDK_CLASSES['TextBlock']
630
+ ToolUseBlock = _SDK_CLASSES['ToolUseBlock']
631
+ ToolResultBlock = _SDK_CLASSES['ToolResultBlock']
632
+
633
+ self.logger.info(
634
+ "starting_claude_code_streaming_execution",
635
+ execution_id=context.execution_id,
636
+ model=context.model_id,
637
+ has_history=bool(context.conversation_history),
638
+ has_callback=event_callback is not None,
639
+ )
640
+
641
+ # Build options and create client
642
+ options, active_tools, started_tools, completed_tools = await build_claude_options(context, event_callback, runtime=self)
643
+
644
+ # Merge custom MCP servers
645
+ if self._custom_mcp_servers:
646
+ if not options.mcp_servers:
647
+ options.mcp_servers = {}
648
+
649
+ for server_name, mcp_server in self._custom_mcp_servers.items():
650
+ options.mcp_servers[server_name] = mcp_server
651
+
652
+ # Add tool names to allowed_tools for permission
653
+ if hasattr(mcp_server, 'tools') and mcp_server.tools:
654
+ for tool in mcp_server.tools:
655
+ if hasattr(tool, 'name'):
656
+ tool_name = f"mcp__{server_name}__{tool.name}"
657
+ if tool_name not in options.allowed_tools:
658
+ options.allowed_tools.append(tool_name)
659
+
660
+ self.logger.debug(
661
+ "custom_mcp_server_added_streaming",
662
+ server_name=server_name,
663
+ execution_id=context.execution_id
664
+ )
665
+
666
+ self.logger.info(
667
+ "created_claude_code_sdk_options",
668
+ execution_id=context.execution_id,
669
+ has_tools=bool(context.skills),
670
+ has_mcp=(
671
+ len(options.mcp_servers) > 0
672
+ if hasattr(options, "mcp_servers")
673
+ else False
674
+ ),
675
+ has_custom_mcp=len(self._custom_mcp_servers) > 0,
676
+ has_hooks=bool(options.hooks) if hasattr(options, "hooks") else False,
677
+ has_event_callback=event_callback is not None,
678
+ )
679
+
680
+ # Suppress verbose MCP STDIO parsing error logs
681
+ # These errors occur when MCP servers incorrectly log to stdout instead of stderr
682
+ # The connection continues to work, but the SDK logs errors for each non-JSONRPC line
683
+ import logging
684
+ mcp_stdio_logger = logging.getLogger("mcp.client.stdio")
685
+ original_stdio_level = mcp_stdio_logger.level
686
+ mcp_stdio_logger.setLevel(logging.ERROR) # Only show critical errors
687
+
688
+ # Create and connect client
689
+ client = ClaudeSDKClient(options=options)
690
+
691
+ try:
692
+ await client.connect()
693
+ except Exception as e:
694
+ error_msg = str(e)
695
+
696
+ # Build comprehensive error message from all sources
697
+ # The SDK wraps subprocess errors, so "No conversation found" may be in:
698
+ # 1. The exception message itself
699
+ # 2. The stderr attribute (if CalledProcessError)
700
+ # 3. The output attribute
701
+ # 4. The __cause__ chain
702
+ full_error_context = error_msg
703
+ if hasattr(e, 'stderr') and e.stderr:
704
+ stderr_content = e.stderr if isinstance(e.stderr, str) else e.stderr.decode('utf-8', errors='replace')
705
+ full_error_context += f" | stderr: {stderr_content}"
706
+ if hasattr(e, 'output') and e.output:
707
+ output_content = e.output if isinstance(e.output, str) else e.output.decode('utf-8', errors='replace')
708
+ full_error_context += f" | output: {output_content}"
709
+ if e.__cause__:
710
+ full_error_context += f" | cause: {str(e.__cause__)}"
711
+
712
+ # Detect session-related errors:
713
+ # 1. Explicit "No conversation found" in any error context
714
+ # 2. "exit code 1" when we were trying to resume (likely session issue)
715
+ is_session_error = (
716
+ "No conversation found" in full_error_context or
717
+ "conversation" in full_error_context.lower() or
718
+ # Fallback: exit code 1 during resume attempt is likely a session issue
719
+ (options.resume and "exit code 1" in error_msg.lower())
720
+ )
721
+
722
+ # Handle session-related errors gracefully by retrying without resume
723
+ if is_session_error and options.resume:
724
+ logger.warning(
725
+ "session_resume_failed_retrying_without_resume_streaming",
726
+ execution_id=context.execution_id[:16],
727
+ error=error_msg,
728
+ full_error_context=full_error_context[:500], # Truncate for logging
729
+ session_id=options.resume[:16] if options.resume else None,
730
+ note="Session doesn't exist or is invalid, creating new conversation"
731
+ )
732
+
733
+ # Retry without resume parameter
734
+ options.resume = None
735
+ session_resume_failed = True # Track for prompt building
736
+ client = ClaudeSDKClient(options=options)
737
+ await client.connect()
738
+ logger.info(
739
+ "client_connected_without_resume_streaming",
740
+ execution_id=context.execution_id[:16]
741
+ )
742
+ else:
743
+ # Different error, re-raise
744
+ raise
745
+ else:
746
+ session_resume_failed = False
747
+
748
+ self._active_clients[context.execution_id] = client
749
+
750
+ # Cache execution metadata
751
+ try:
752
+ self.control_plane.cache_metadata(context.execution_id, "AGENT")
753
+ except Exception as cache_error:
754
+ self.logger.warning(
755
+ "failed_to_cache_metadata_non_fatal",
756
+ execution_id=context.execution_id,
757
+ error=str(cache_error),
758
+ )
759
+
760
+ # Send prompt - include history in prompt if session resume failed
761
+ # (since we can't rely on SDK's session continuity)
762
+ if session_resume_failed and context.conversation_history:
763
+ prompt = build_prompt_with_history(context)
764
+ logger.info(
765
+ "using_prompt_with_history_fallback_streaming",
766
+ execution_id=context.execution_id[:16],
767
+ history_messages=len(context.conversation_history),
768
+ note="Session resume failed, including history in prompt"
769
+ )
770
+ else:
771
+ prompt = context.prompt
772
+
773
+ self.logger.debug(
774
+ "sending_streaming_query_to_claude_code_sdk",
775
+ execution_id=context.execution_id,
776
+ prompt_length=len(prompt),
777
+ using_session_resume=bool(options.resume),
778
+ )
779
+
780
+ await client.query(prompt)
781
+
782
+ # Stream messages
783
+ accumulated_response = ""
784
+ accumulated_usage = {}
785
+ tool_messages = []
786
+ message_count = 0
787
+ received_stream_events = False # Track if we got streaming events
788
+ session_id = None # Initialize to avoid UnboundLocalError in exception handlers
789
+ last_heartbeat = time.time() # Track last heartbeat for Temporal activity liveness
790
+
791
+ # completed_tools set is now passed from build_claude_options for tracking
792
+ # which tool_use_ids have published completion events (prevents duplicates and detects missing)
793
+
794
+ # Generate unique message_id for this turn
795
+ message_id = f"{context.execution_id}_{int(time.time() * 1000000)}"
796
+
797
+ # Track thinking state for extended thinking support
798
+ current_thinking_block = None # Tracks active thinking block index
799
+ thinking_content_buffer = "" # Accumulates thinking content for logging
800
+
801
+ async for message in client.receive_response():
802
+ message_count += 1
803
+ message_type_name = type(message).__name__
804
+
805
+ # Handle StreamEvent messages (partial chunks)
806
+ if message_type_name == "StreamEvent":
807
+ if hasattr(message, "event") and message.event:
808
+ event_data = message.event
809
+
810
+ # Extract text from event data
811
+ content = None
812
+ if isinstance(event_data, dict):
813
+ event_type = event_data.get("type")
814
+
815
+ # Handle content_block_start events (detect thinking blocks)
816
+ if event_type == "content_block_start":
817
+ content_block = event_data.get("content_block", {})
818
+ block_type = content_block.get("type")
819
+ block_index = event_data.get("index", 0)
820
+
821
+ if block_type == "thinking":
822
+ current_thinking_block = block_index
823
+ thinking_content_buffer = ""
824
+
825
+ # Emit thinking_start event
826
+ if event_callback:
827
+ try:
828
+ event_callback({
829
+ "type": "thinking_start",
830
+ "message_id": message_id,
831
+ "index": block_index,
832
+ "execution_id": context.execution_id,
833
+ })
834
+ except Exception as callback_error:
835
+ self.logger.warning(
836
+ "thinking_start_callback_failed",
837
+ execution_id=context.execution_id,
838
+ error=str(callback_error),
839
+ )
840
+ continue # Don't process content_block_start further
841
+
842
+ # Handle content_block_delta events
843
+ if event_type == "content_block_delta":
844
+ delta = event_data.get("delta", {})
845
+ delta_type = delta.get("type") if isinstance(delta, dict) else None
846
+
847
+ # Handle thinking_delta events
848
+ if delta_type == "thinking_delta":
849
+ thinking_text = delta.get("thinking", "")
850
+ if thinking_text:
851
+ thinking_content_buffer += thinking_text
852
+
853
+ # Emit thinking_delta event
854
+ if event_callback:
855
+ try:
856
+ event_callback({
857
+ "type": "thinking_delta",
858
+ "thinking": thinking_text,
859
+ "message_id": message_id,
860
+ "index": event_data.get("index", 0),
861
+ "execution_id": context.execution_id,
862
+ })
863
+ except Exception as callback_error:
864
+ self.logger.warning(
865
+ "thinking_delta_callback_failed",
866
+ execution_id=context.execution_id,
867
+ error=str(callback_error),
868
+ )
869
+ continue # Don't process thinking as regular content
870
+
871
+ # Handle signature_delta events (end of thinking block)
872
+ if delta_type == "signature_delta":
873
+ signature = delta.get("signature", "")
874
+
875
+ # Emit thinking_complete event
876
+ if event_callback:
877
+ try:
878
+ event_callback({
879
+ "type": "thinking_complete",
880
+ "signature": signature,
881
+ "message_id": message_id,
882
+ "index": event_data.get("index", 0),
883
+ "execution_id": context.execution_id,
884
+ })
885
+ except Exception as callback_error:
886
+ self.logger.warning(
887
+ "thinking_complete_callback_failed",
888
+ execution_id=context.execution_id,
889
+ error=str(callback_error),
890
+ )
891
+
892
+ # Log thinking summary
893
+ if thinking_content_buffer:
894
+ self.logger.debug(
895
+ "thinking_block_completed",
896
+ execution_id=context.execution_id,
897
+ thinking_length=len(thinking_content_buffer),
898
+ has_signature=bool(signature),
899
+ )
900
+ current_thinking_block = None
901
+ thinking_content_buffer = ""
902
+ continue # Don't process signature as regular content
903
+
904
+ # Handle text_delta events (regular text content)
905
+ if isinstance(delta, dict):
906
+ content = delta.get("text")
907
+ elif isinstance(delta, str):
908
+ content = delta
909
+
910
+ # Fallback: try direct text extraction
911
+ if not content:
912
+ content = event_data.get("text") or event_data.get(
913
+ "content"
914
+ )
915
+
916
+ elif isinstance(event_data, str):
917
+ content = event_data
918
+ elif hasattr(event_data, "content"):
919
+ content = event_data.content
920
+ elif hasattr(event_data, "text"):
921
+ content = event_data.text
922
+
923
+ if content:
924
+ received_stream_events = True
925
+ chunk_count += 1
926
+ accumulated_response += content
927
+
928
+ # Publish event
929
+ if event_callback:
930
+ try:
931
+ event_callback(
932
+ {
933
+ "type": "content_chunk",
934
+ "content": content,
935
+ "message_id": message_id,
936
+ "execution_id": context.execution_id,
937
+ }
938
+ )
939
+ except Exception as callback_error:
940
+ self.logger.warning(
941
+ "stream_event_callback_failed",
942
+ execution_id=context.execution_id,
943
+ error=str(callback_error),
944
+ )
945
+
946
+ # Yield chunk with explicit empty arrays for frontend compatibility
947
+ # Frontend expects arrays, not None, to avoid R.map errors
948
+ yield RuntimeExecutionResult(
949
+ response=content,
950
+ usage={},
951
+ success=True,
952
+ tool_messages=[],
953
+ tool_execution_messages=[],
954
+ )
955
+
956
+ # Send heartbeat every 10 seconds or every 50 chunks (matches AgnoRuntime pattern)
957
+ current_time = time.time()
958
+ if current_time - last_heartbeat > 10 or chunk_count % 50 == 0:
959
+ try:
960
+ activity.heartbeat({
961
+ "status": "streaming",
962
+ "chunks_received": chunk_count,
963
+ "response_length": len(accumulated_response),
964
+ "elapsed_seconds": int(current_time - last_heartbeat)
965
+ })
966
+ last_heartbeat = current_time
967
+ except Exception as e:
968
+ # Non-fatal: heartbeat failure should not break execution
969
+ self.logger.warning("heartbeat_failed_non_fatal", execution_id=context.execution_id, error=str(e))
970
+
971
+ continue # Skip to next message
972
+
973
+ # Handle assistant messages (final complete message)
974
+ if isinstance(message, AssistantMessage):
975
+ for block in message.content:
976
+ if isinstance(block, TextBlock):
977
+ # Skip if already streamed via StreamEvents
978
+ if received_stream_events:
979
+ continue
980
+
981
+ # Only send if we didn't receive StreamEvents
982
+ chunk_count += 1
983
+ accumulated_response += block.text
984
+
985
+ if event_callback:
986
+ try:
987
+ event_callback(
988
+ {
989
+ "type": "content_chunk",
990
+ "content": block.text,
991
+ "message_id": message_id,
992
+ "execution_id": context.execution_id,
993
+ }
994
+ )
995
+ except Exception as callback_error:
996
+ self.logger.warning(
997
+ "event_callback_failed_non_fatal",
998
+ execution_id=context.execution_id,
999
+ error=str(callback_error),
1000
+ )
1001
+
1002
+ # Frontend expects arrays, not None, to avoid R.map errors
1003
+ yield RuntimeExecutionResult(
1004
+ response=block.text,
1005
+ usage={},
1006
+ success=True,
1007
+ tool_messages=[],
1008
+ tool_execution_messages=[],
1009
+ )
1010
+
1011
+ # Send heartbeat every 10 seconds or every 50 chunks (matches AgnoRuntime pattern)
1012
+ current_time = time.time()
1013
+ if current_time - last_heartbeat > 10 or chunk_count % 50 == 0:
1014
+ try:
1015
+ activity.heartbeat({
1016
+ "status": "streaming",
1017
+ "chunks_received": chunk_count,
1018
+ "response_length": len(accumulated_response),
1019
+ "elapsed_seconds": int(current_time - last_heartbeat)
1020
+ })
1021
+ last_heartbeat = current_time
1022
+ except Exception as e:
1023
+ # Non-fatal: heartbeat failure should not break execution
1024
+ self.logger.warning("heartbeat_failed_non_fatal", execution_id=context.execution_id, error=str(e))
1025
+
1026
+ elif isinstance(block, ToolUseBlock):
1027
+ # Tool use event - Store for later lookup
1028
+ tool_info = {
1029
+ "tool": block.name,
1030
+ "input": block.input,
1031
+ "tool_use_id": block.id,
1032
+ }
1033
+ tool_messages.append(tool_info)
1034
+ active_tools[block.id] = block.name
1035
+
1036
+ # Publish tool_start event from ToolUseBlock (with deduplication)
1037
+ # This ensures built-in tools like TodoWrite that skip hooks still get tool_start events
1038
+ if event_callback and block.id not in started_tools:
1039
+ try:
1040
+ event_callback(
1041
+ {
1042
+ "type": "tool_start",
1043
+ "tool_name": block.name,
1044
+ "tool_args": block.input, # Include tool input for frontend rendering
1045
+ "tool_execution_id": block.id,
1046
+ "execution_id": context.execution_id,
1047
+ }
1048
+ )
1049
+ started_tools.add(block.id)
1050
+ self.logger.debug(
1051
+ "tool_start_published_via_stream",
1052
+ tool_use_id=block.id,
1053
+ tool_name=block.name,
1054
+ )
1055
+ except Exception as callback_error:
1056
+ self.logger.error(
1057
+ "failed_to_publish_tool_start_from_stream",
1058
+ tool_use_id=block.id,
1059
+ tool_name=block.name,
1060
+ error=str(callback_error),
1061
+ )
1062
+
1063
+ elif isinstance(block, ToolResultBlock):
1064
+ # Tool result - Look up tool name from active_tools
1065
+ tool_name = active_tools.get(block.tool_use_id, "unknown")
1066
+ if tool_name == "unknown":
1067
+ self.logger.warning(
1068
+ "could_not_find_tool_name_for_tool_use_id",
1069
+ execution_id=context.execution_id,
1070
+ tool_use_id=block.tool_use_id,
1071
+ active_tools_keys=list(active_tools.keys()),
1072
+ )
1073
+
1074
+ status = "success" if not block.is_error else "failed"
1075
+
1076
+ # Publish via callback (with deduplication)
1077
+ if event_callback and block.tool_use_id not in completed_tools:
1078
+ try:
1079
+ event_callback(
1080
+ {
1081
+ "type": "tool_complete",
1082
+ "tool_name": tool_name,
1083
+ "tool_execution_id": block.tool_use_id,
1084
+ "status": status,
1085
+ "output": (
1086
+ str(block.content)[:1000]
1087
+ if block.content
1088
+ else None
1089
+ ),
1090
+ "error": (
1091
+ str(block.content)
1092
+ if block.is_error
1093
+ else None
1094
+ ),
1095
+ "execution_id": context.execution_id,
1096
+ }
1097
+ )
1098
+ # Mark as completed to prevent duplicate events
1099
+ completed_tools.add(block.tool_use_id)
1100
+ self.logger.debug(
1101
+ "tool_complete_published_via_stream",
1102
+ tool_use_id=block.tool_use_id,
1103
+ tool_name=tool_name,
1104
+ )
1105
+ except Exception as callback_error:
1106
+ self.logger.error(
1107
+ "tool_complete_callback_failed",
1108
+ execution_id=context.execution_id,
1109
+ tool_name=tool_name,
1110
+ error=str(callback_error),
1111
+ exc_info=True,
1112
+ )
1113
+ elif block.tool_use_id in completed_tools:
1114
+ self.logger.debug(
1115
+ "tool_complete_already_published_via_hooks",
1116
+ tool_use_id=block.tool_use_id,
1117
+ tool_name=tool_name,
1118
+ )
1119
+
1120
+ # Handle result message (final)
1121
+ elif isinstance(message, ResultMessage):
1122
+ accumulated_usage = extract_usage_from_result_message(message)
1123
+
1124
+ # BUG FIX #4: Extract and validate session_id
1125
+ session_id = extract_session_id_from_result_message(
1126
+ message, context.execution_id
1127
+ )
1128
+
1129
+ elapsed_time = asyncio.get_event_loop().time() - start_time
1130
+
1131
+ # FALLBACK: Detect missing tool completion events
1132
+ # Check if any tool_use_ids in tool_messages are not in completed_tools
1133
+ missing_completions = []
1134
+ for tool_info in tool_messages:
1135
+ tool_use_id = tool_info.get("tool_use_id")
1136
+ if tool_use_id and tool_use_id not in completed_tools:
1137
+ missing_completions.append(tool_info)
1138
+
1139
+ if missing_completions:
1140
+ # Categorize missing tools by type for better diagnostics
1141
+ task_tools = [t for t in missing_completions if t.get("tool") == "Task"]
1142
+ builtin_tools = [t for t in missing_completions if t.get("tool") in ["TodoWrite", "Bash", "Read", "Write", "Edit", "Glob", "Grep"]]
1143
+ other_tools = [t for t in missing_completions if t not in task_tools and t not in builtin_tools]
1144
+
1145
+ # Use warning level only if unexpected tools are missing
1146
+ # Task and builtin tools are expected to miss hooks sometimes
1147
+ log_level = "warning" if other_tools else "info"
1148
+
1149
+ log_message = (
1150
+ f"Publishing fallback completion events for {len(missing_completions)} tools. "
1151
+ f"Task tools: {len(task_tools)} (expected - subagents execute in separate contexts), "
1152
+ f"Built-in tools: {len(builtin_tools)} (expected - may use optimized execution paths), "
1153
+ f"Other: {len(other_tools)} (unexpected - may indicate hook registration issue)"
1154
+ )
1155
+
1156
+ getattr(self.logger, log_level)(
1157
+ "detected_missing_tool_completion_events",
1158
+ execution_id=context.execution_id,
1159
+ missing_count=len(missing_completions),
1160
+ task_tools_count=len(task_tools),
1161
+ builtin_tools_count=len(builtin_tools),
1162
+ other_tools_count=len(other_tools),
1163
+ missing_tool_names=[t.get("tool") for t in missing_completions],
1164
+ missing_tool_ids=[t.get("tool_use_id")[:12] for t in missing_completions],
1165
+ task_tool_ids=[t.get("tool_use_id")[:12] for t in task_tools] if task_tools else [],
1166
+ builtin_tool_ids=[t.get("tool_use_id")[:12] for t in builtin_tools] if builtin_tools else [],
1167
+ other_tool_ids=[t.get("tool_use_id")[:12] for t in other_tools] if other_tools else [],
1168
+ message=log_message,
1169
+ note=(
1170
+ "SubagentStop hook should reduce Task tool misses. "
1171
+ "Built-in tools may skip hooks by design. "
1172
+ "Investigate 'other' category if count > 0."
1173
+ )
1174
+ )
1175
+
1176
+ # Publish missing completion events
1177
+ if event_callback:
1178
+ for tool_info in missing_completions:
1179
+ # Defensive: Validate tool_info has required fields
1180
+ if not isinstance(tool_info, dict):
1181
+ self.logger.warning(
1182
+ "invalid_tool_info_type_in_fallback",
1183
+ tool_info_type=type(tool_info).__name__,
1184
+ note="Skipping invalid tool_info"
1185
+ )
1186
+ continue
1187
+
1188
+ tool_use_id = tool_info.get("tool_use_id")
1189
+ tool_name = tool_info.get("tool", "unknown")
1190
+
1191
+ # Defensive: Skip if no tool_use_id
1192
+ if not tool_use_id:
1193
+ self.logger.warning(
1194
+ "missing_tool_use_id_in_fallback",
1195
+ tool_name=tool_name,
1196
+ note="Cannot publish completion without tool_use_id"
1197
+ )
1198
+ continue
1199
+
1200
+ try:
1201
+ # Defensive: Safe slicing for logging
1202
+ tool_use_id_short = (
1203
+ tool_use_id[:12] if isinstance(tool_use_id, str) and len(tool_use_id) >= 12
1204
+ else str(tool_use_id)
1205
+ )
1206
+
1207
+ event_callback(
1208
+ {
1209
+ "type": "tool_complete",
1210
+ "tool_name": tool_name,
1211
+ "tool_execution_id": tool_use_id,
1212
+ "status": "success", # Assume success if no error was caught
1213
+ "output": None, # No output available in fallback
1214
+ "error": None,
1215
+ "execution_id": context.execution_id,
1216
+ }
1217
+ )
1218
+ completed_tools.add(tool_use_id)
1219
+ self.logger.info(
1220
+ "published_fallback_tool_completion",
1221
+ tool_use_id=tool_use_id_short,
1222
+ tool_name=tool_name,
1223
+ note="Fallback completion event published successfully"
1224
+ )
1225
+ except Exception as e:
1226
+ # Non-fatal: Log but continue processing other tools
1227
+ self.logger.error(
1228
+ "failed_to_publish_fallback_completion",
1229
+ tool_use_id=str(tool_use_id) if tool_use_id else "unknown",
1230
+ tool_name=tool_name,
1231
+ error=str(e),
1232
+ error_type=type(e).__name__,
1233
+ exc_info=True,
1234
+ note="Continuing with remaining tools despite failure"
1235
+ )
1236
+
1237
+ self.logger.info(
1238
+ "claude_code_streaming_completed",
1239
+ execution_id=context.execution_id,
1240
+ finish_reason=message.subtype,
1241
+ chunk_count=chunk_count,
1242
+ message_count=message_count,
1243
+ response_length=len(accumulated_response),
1244
+ tool_count=len(tool_messages),
1245
+ completed_tool_count=len(completed_tools),
1246
+ missing_completions=len(missing_completions) if missing_completions else 0,
1247
+ usage=accumulated_usage,
1248
+ elapsed_time=f"{elapsed_time:.2f}s",
1249
+ has_session_id=bool(session_id),
1250
+ )
1251
+
1252
+ # Final result message
1253
+ yield RuntimeExecutionResult(
1254
+ response="", # Already streamed
1255
+ usage=accumulated_usage,
1256
+ success=message.subtype == "success",
1257
+ finish_reason=message.subtype,
1258
+ tool_execution_messages=tool_messages, # Use standard field name for analytics
1259
+ tool_messages=tool_messages, # Keep for backward compatibility
1260
+ model=context.model_id,
1261
+ metadata={
1262
+ "accumulated_response": accumulated_response,
1263
+ "elapsed_time": elapsed_time,
1264
+ "chunk_count": chunk_count,
1265
+ "message_count": message_count,
1266
+ "claude_code_session_id": session_id,
1267
+ },
1268
+ )
1269
+ break
1270
+
1271
+ except ImportError as e:
1272
+ elapsed_time = asyncio.get_event_loop().time() - start_time
1273
+ self.logger.error(
1274
+ "claude_code_sdk_not_installed",
1275
+ execution_id=context.execution_id,
1276
+ error=str(e),
1277
+ elapsed_time=f"{elapsed_time:.2f}s",
1278
+ )
1279
+ yield RuntimeExecutionResult(
1280
+ response="",
1281
+ usage={},
1282
+ success=False,
1283
+ error=f"Claude Code SDK not available: {str(e)}",
1284
+ tool_messages=[],
1285
+ tool_execution_messages=[],
1286
+ )
1287
+
1288
+ except asyncio.TimeoutError:
1289
+ elapsed_time = asyncio.get_event_loop().time() - start_time
1290
+ self.logger.error(
1291
+ "claude_code_streaming_timeout",
1292
+ execution_id=context.execution_id,
1293
+ elapsed_time=f"{elapsed_time:.2f}s",
1294
+ chunks_before_timeout=chunk_count,
1295
+ )
1296
+ yield RuntimeExecutionResult(
1297
+ response="",
1298
+ usage={},
1299
+ success=False,
1300
+ error="Streaming execution timeout exceeded",
1301
+ tool_messages=[],
1302
+ tool_execution_messages=[],
1303
+ )
1304
+
1305
+ except asyncio.CancelledError:
1306
+ elapsed_time = asyncio.get_event_loop().time() - start_time
1307
+ self.logger.warning(
1308
+ "claude_code_streaming_cancelled_gracefully",
1309
+ execution_id=context.execution_id,
1310
+ elapsed_time=f"{elapsed_time:.2f}s",
1311
+ chunks_before_cancellation=chunk_count,
1312
+ accumulated_response_length=len(accumulated_response),
1313
+ session_id=session_id[:16] if session_id else None,
1314
+ )
1315
+
1316
+ # DURABILITY FIX: Do NOT re-raise! Handle cancellation gracefully
1317
+ # Save partial state and allow workflow to resume from here
1318
+ # The workflow is durable and should handle interruptions
1319
+
1320
+ # Yield partial success result with accumulated state
1321
+ yield RuntimeExecutionResult(
1322
+ response=accumulated_response, # Return what we accumulated so far
1323
+ usage=accumulated_usage,
1324
+ success=True, # Partial success, not a failure
1325
+ finish_reason="cancelled",
1326
+ tool_execution_messages=tool_messages,
1327
+ tool_messages=tool_messages,
1328
+ model=context.model_id,
1329
+ metadata={
1330
+ "accumulated_response": accumulated_response,
1331
+ "elapsed_time": elapsed_time,
1332
+ "chunk_count": chunk_count,
1333
+ "message_count": message_count,
1334
+ "claude_code_session_id": session_id,
1335
+ "interrupted": True, # Flag that this was interrupted
1336
+ "can_resume": bool(session_id), # Can resume if we have session_id
1337
+ },
1338
+ )
1339
+ # NOTE: Do NOT re-raise - this would break Temporal durability!
1340
+
1341
+ except Exception as e:
1342
+ elapsed_time = asyncio.get_event_loop().time() - start_time
1343
+ self.logger.error(
1344
+ "claude_code_streaming_failed",
1345
+ execution_id=context.execution_id,
1346
+ error=str(e),
1347
+ error_type=type(e).__name__,
1348
+ elapsed_time=f"{elapsed_time:.2f}s",
1349
+ chunks_before_error=chunk_count,
1350
+ exc_info=True,
1351
+ )
1352
+ yield RuntimeExecutionResult(
1353
+ response="",
1354
+ usage={},
1355
+ success=False,
1356
+ error=f"{type(e).__name__}: {str(e)}",
1357
+ finish_reason="error", # CRITICAL: Must set finish_reason so caller recognizes this as final result
1358
+ tool_messages=[],
1359
+ tool_execution_messages=[],
1360
+ )
1361
+
1362
+ finally:
1363
+ # Clear execution context from proxy (with delay to allow in-flight SDK requests)
1364
+ try:
1365
+ clear_execution_context(
1366
+ context.execution_id,
1367
+ immediate=False, # Use delayed cleanup
1368
+ delay_seconds=5.0 # Wait for in-flight SDK requests
1369
+ )
1370
+ except Exception as e:
1371
+ self.logger.warning(
1372
+ "failed_to_clear_proxy_context_streaming",
1373
+ execution_id=context.execution_id,
1374
+ error=str(e),
1375
+ )
1376
+
1377
+ # Restore MCP STDIO log level
1378
+ try:
1379
+ import logging
1380
+ mcp_stdio_logger = logging.getLogger("mcp.client.stdio")
1381
+ if 'original_stdio_level' in locals():
1382
+ mcp_stdio_logger.setLevel(original_stdio_level)
1383
+ except Exception as log_level_error:
1384
+ # Log but ignore errors restoring log level - this is non-critical cleanup
1385
+ self.logger.debug(
1386
+ "failed_to_restore_log_level",
1387
+ error=str(log_level_error),
1388
+ execution_id=context.execution_id
1389
+ )
1390
+
1391
+ # CRITICAL: Cleanup SDK client
1392
+ if context.execution_id in self._active_clients:
1393
+ client = self._active_clients.pop(context.execution_id)
1394
+ cleanup_sdk_client(client, context.execution_id, self.logger)
1395
+
1396
+ async def cancel(self, execution_id: str) -> bool:
1397
+ """
1398
+ Cancel an in-progress execution via Claude SDK interrupt.
1399
+
1400
+ Args:
1401
+ execution_id: ID of execution to cancel
1402
+
1403
+ Returns:
1404
+ True if cancellation succeeded
1405
+ """
1406
+ if execution_id in self._active_clients:
1407
+ try:
1408
+ client = self._active_clients[execution_id]
1409
+ await client.interrupt()
1410
+ self.logger.info(
1411
+ "claude_code_execution_interrupted", execution_id=execution_id
1412
+ )
1413
+ return True
1414
+ except Exception as e:
1415
+ self.logger.error(
1416
+ "failed_to_interrupt_claude_code_execution",
1417
+ execution_id=execution_id,
1418
+ error=str(e),
1419
+ )
1420
+ return False
1421
+ return False
1422
+
1423
+ # ==================== Custom Tool Extension API ====================
1424
+
1425
+ def get_custom_tool_requirements(self) -> Dict[str, Any]:
1426
+ """
1427
+ Get requirements for creating custom MCP servers for Claude Code runtime.
1428
+
1429
+ Returns:
1430
+ Dictionary with format, examples, and documentation for MCP servers
1431
+ """
1432
+ return {
1433
+ "format": "mcp_server",
1434
+ "description": "MCP server created with @tool decorator and create_sdk_mcp_server()",
1435
+ "example_code": '''
1436
+ from claude_agent_sdk import tool, create_sdk_mcp_server
1437
+ from typing import Any
1438
+
1439
+ @tool("my_function", "Description of what this tool does", {"arg": str})
1440
+ async def my_function(args: dict[str, Any]) -> dict[str, Any]:
1441
+ """Tool function implementation."""
1442
+ return {
1443
+ "content": [{
1444
+ "type": "text",
1445
+ "text": f"Result: {args['arg']}"
1446
+ }]
1447
+ }
1448
+
1449
+ # Create MCP server
1450
+ mcp_server = create_sdk_mcp_server(
1451
+ name="my_tools",
1452
+ version="1.0.0",
1453
+ tools=[my_function]
1454
+ )
1455
+ ''',
1456
+ "documentation_url": "https://docs.claude.ai/agent-sdk/custom-tools",
1457
+ "required_attributes": ["name", "version"],
1458
+ "schema": {
1459
+ "type": "mcp_server",
1460
+ "required": ["name", "version", "tools"]
1461
+ }
1462
+ }
1463
+
1464
+ def validate_custom_tool(self, mcp_server: Any) -> tuple[bool, Optional[str]]:
1465
+ """
1466
+ Validate an MCP server for Claude Code runtime.
1467
+
1468
+ Args:
1469
+ mcp_server: MCP server instance to validate
1470
+
1471
+ Returns:
1472
+ Tuple of (is_valid, error_message)
1473
+ """
1474
+ # Check required attributes
1475
+ for attr in ['name', 'version']:
1476
+ if not hasattr(mcp_server, attr):
1477
+ return False, f"MCP server must have '{attr}' attribute"
1478
+
1479
+ # Validate name
1480
+ if not isinstance(mcp_server.name, str) or not mcp_server.name:
1481
+ return False, "MCP server name must be non-empty string"
1482
+
1483
+ # Check for tools (optional but recommended)
1484
+ if hasattr(mcp_server, 'tools'):
1485
+ if not mcp_server.tools:
1486
+ self.logger.warning(
1487
+ "mcp_server_has_no_tools",
1488
+ server_name=mcp_server.name
1489
+ )
1490
+
1491
+ return True, None
1492
+
1493
+ def register_custom_tool(self, mcp_server: Any, metadata: Optional[Dict] = None) -> str:
1494
+ """
1495
+ Register a custom MCP server with Claude Code runtime.
1496
+
1497
+ Args:
1498
+ mcp_server: MCP server instance
1499
+ metadata: Optional metadata (ignored, server name is used)
1500
+
1501
+ Returns:
1502
+ Server name (identifier for this MCP server)
1503
+
1504
+ Raises:
1505
+ ValueError: If MCP server validation fails or name conflicts
1506
+ """
1507
+ # Validate first
1508
+ is_valid, error = self.validate_custom_tool(mcp_server)
1509
+ if not is_valid:
1510
+ raise ValueError(f"Invalid MCP server: {error}")
1511
+
1512
+ server_name = mcp_server.name
1513
+
1514
+ # Check for name conflicts
1515
+ if server_name in self._custom_mcp_servers:
1516
+ raise ValueError(f"MCP server '{server_name}' already registered")
1517
+
1518
+ # Store MCP server
1519
+ self._custom_mcp_servers[server_name] = mcp_server
1520
+
1521
+ # Extract tool names for logging
1522
+ tool_names = []
1523
+ if hasattr(mcp_server, 'tools') and mcp_server.tools:
1524
+ tool_names = [
1525
+ f"mcp__{server_name}__{t.name}"
1526
+ for t in mcp_server.tools
1527
+ if hasattr(t, 'name')
1528
+ ]
1529
+
1530
+ self.logger.info(
1531
+ "custom_mcp_server_registered",
1532
+ server_name=server_name,
1533
+ tool_count=len(tool_names),
1534
+ tools=tool_names
1535
+ )
1536
+
1537
+ return server_name
1538
+
1539
+ def get_registered_custom_tools(self) -> list[str]:
1540
+ """
1541
+ Get list of registered custom MCP server names.
1542
+
1543
+ Returns:
1544
+ List of server names
1545
+ """
1546
+ return list(self._custom_mcp_servers.keys())