autobyteus 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (466) hide show
  1. autobyteus/agent/agent.py +15 -5
  2. autobyteus/agent/bootstrap_steps/__init__.py +1 -3
  3. autobyteus/agent/bootstrap_steps/agent_bootstrapper.py +3 -59
  4. autobyteus/agent/bootstrap_steps/base_bootstrap_step.py +1 -4
  5. autobyteus/agent/bootstrap_steps/mcp_server_prewarming_step.py +1 -3
  6. autobyteus/agent/bootstrap_steps/system_prompt_processing_step.py +16 -13
  7. autobyteus/agent/bootstrap_steps/workspace_context_initialization_step.py +2 -4
  8. autobyteus/agent/context/agent_config.py +43 -20
  9. autobyteus/agent/context/agent_context.py +23 -18
  10. autobyteus/agent/context/agent_runtime_state.py +19 -19
  11. autobyteus/agent/events/__init__.py +16 -1
  12. autobyteus/agent/events/agent_events.py +43 -3
  13. autobyteus/agent/events/agent_input_event_queue_manager.py +79 -26
  14. autobyteus/agent/events/event_store.py +57 -0
  15. autobyteus/agent/events/notifiers.py +69 -59
  16. autobyteus/agent/events/worker_event_dispatcher.py +21 -64
  17. autobyteus/agent/factory/agent_factory.py +52 -0
  18. autobyteus/agent/handlers/__init__.py +2 -0
  19. autobyteus/agent/handlers/approved_tool_invocation_event_handler.py +51 -34
  20. autobyteus/agent/handlers/bootstrap_event_handler.py +155 -0
  21. autobyteus/agent/handlers/inter_agent_message_event_handler.py +10 -0
  22. autobyteus/agent/handlers/lifecycle_event_logger.py +19 -11
  23. autobyteus/agent/handlers/llm_complete_response_received_event_handler.py +10 -15
  24. autobyteus/agent/handlers/llm_user_message_ready_event_handler.py +188 -48
  25. autobyteus/agent/handlers/tool_execution_approval_event_handler.py +0 -10
  26. autobyteus/agent/handlers/tool_invocation_request_event_handler.py +53 -48
  27. autobyteus/agent/handlers/tool_result_event_handler.py +7 -8
  28. autobyteus/agent/handlers/user_input_message_event_handler.py +10 -3
  29. autobyteus/agent/input_processor/memory_ingest_input_processor.py +40 -0
  30. autobyteus/agent/lifecycle/__init__.py +12 -0
  31. autobyteus/agent/lifecycle/base_processor.py +109 -0
  32. autobyteus/agent/lifecycle/events.py +35 -0
  33. autobyteus/agent/lifecycle/processor_definition.py +36 -0
  34. autobyteus/agent/lifecycle/processor_registry.py +106 -0
  35. autobyteus/agent/llm_request_assembler.py +98 -0
  36. autobyteus/agent/llm_response_processor/__init__.py +1 -8
  37. autobyteus/agent/message/context_file_type.py +1 -1
  38. autobyteus/agent/runtime/agent_runtime.py +29 -21
  39. autobyteus/agent/runtime/agent_worker.py +98 -19
  40. autobyteus/agent/shutdown_steps/__init__.py +2 -0
  41. autobyteus/agent/shutdown_steps/agent_shutdown_orchestrator.py +2 -0
  42. autobyteus/agent/shutdown_steps/tool_cleanup_step.py +58 -0
  43. autobyteus/agent/status/__init__.py +14 -0
  44. autobyteus/agent/status/manager.py +93 -0
  45. autobyteus/agent/status/status_deriver.py +96 -0
  46. autobyteus/agent/{phases/phase_enum.py → status/status_enum.py} +16 -16
  47. autobyteus/agent/status/status_update_utils.py +73 -0
  48. autobyteus/agent/streaming/__init__.py +52 -5
  49. autobyteus/agent/streaming/adapters/__init__.py +18 -0
  50. autobyteus/agent/streaming/adapters/invocation_adapter.py +184 -0
  51. autobyteus/agent/streaming/adapters/tool_call_parsing.py +163 -0
  52. autobyteus/agent/streaming/adapters/tool_syntax_registry.py +67 -0
  53. autobyteus/agent/streaming/agent_event_stream.py +3 -183
  54. autobyteus/agent/streaming/api_tool_call/__init__.py +16 -0
  55. autobyteus/agent/streaming/api_tool_call/file_content_streamer.py +56 -0
  56. autobyteus/agent/streaming/api_tool_call/json_string_field_extractor.py +175 -0
  57. autobyteus/agent/streaming/api_tool_call_streaming_response_handler.py +4 -0
  58. autobyteus/agent/streaming/events/__init__.py +6 -0
  59. autobyteus/agent/streaming/events/stream_event_payloads.py +284 -0
  60. autobyteus/agent/streaming/events/stream_events.py +141 -0
  61. autobyteus/agent/streaming/handlers/__init__.py +15 -0
  62. autobyteus/agent/streaming/handlers/api_tool_call_streaming_response_handler.py +303 -0
  63. autobyteus/agent/streaming/handlers/parsing_streaming_response_handler.py +107 -0
  64. autobyteus/agent/streaming/handlers/pass_through_streaming_response_handler.py +107 -0
  65. autobyteus/agent/streaming/handlers/streaming_handler_factory.py +177 -0
  66. autobyteus/agent/streaming/handlers/streaming_response_handler.py +58 -0
  67. autobyteus/agent/streaming/parser/__init__.py +61 -0
  68. autobyteus/agent/streaming/parser/event_emitter.py +181 -0
  69. autobyteus/agent/streaming/parser/events.py +4 -0
  70. autobyteus/agent/streaming/parser/invocation_adapter.py +4 -0
  71. autobyteus/agent/streaming/parser/json_parsing_strategies/__init__.py +19 -0
  72. autobyteus/agent/streaming/parser/json_parsing_strategies/base.py +32 -0
  73. autobyteus/agent/streaming/parser/json_parsing_strategies/default.py +34 -0
  74. autobyteus/agent/streaming/parser/json_parsing_strategies/gemini.py +31 -0
  75. autobyteus/agent/streaming/parser/json_parsing_strategies/openai.py +64 -0
  76. autobyteus/agent/streaming/parser/json_parsing_strategies/registry.py +75 -0
  77. autobyteus/agent/streaming/parser/parser_context.py +227 -0
  78. autobyteus/agent/streaming/parser/parser_factory.py +132 -0
  79. autobyteus/agent/streaming/parser/sentinel_format.py +7 -0
  80. autobyteus/agent/streaming/parser/state_factory.py +62 -0
  81. autobyteus/agent/streaming/parser/states/__init__.py +1 -0
  82. autobyteus/agent/streaming/parser/states/base_state.py +60 -0
  83. autobyteus/agent/streaming/parser/states/custom_xml_tag_run_bash_parsing_state.py +38 -0
  84. autobyteus/agent/streaming/parser/states/custom_xml_tag_write_file_parsing_state.py +55 -0
  85. autobyteus/agent/streaming/parser/states/delimited_content_state.py +146 -0
  86. autobyteus/agent/streaming/parser/states/json_initialization_state.py +144 -0
  87. autobyteus/agent/streaming/parser/states/json_tool_parsing_state.py +137 -0
  88. autobyteus/agent/streaming/parser/states/sentinel_content_state.py +30 -0
  89. autobyteus/agent/streaming/parser/states/sentinel_initialization_state.py +117 -0
  90. autobyteus/agent/streaming/parser/states/text_state.py +78 -0
  91. autobyteus/agent/streaming/parser/states/xml_patch_file_tool_parsing_state.py +328 -0
  92. autobyteus/agent/streaming/parser/states/xml_run_bash_tool_parsing_state.py +129 -0
  93. autobyteus/agent/streaming/parser/states/xml_tag_initialization_state.py +151 -0
  94. autobyteus/agent/streaming/parser/states/xml_tool_parsing_state.py +63 -0
  95. autobyteus/agent/streaming/parser/states/xml_write_file_tool_parsing_state.py +343 -0
  96. autobyteus/agent/streaming/parser/strategies/__init__.py +17 -0
  97. autobyteus/agent/streaming/parser/strategies/base.py +24 -0
  98. autobyteus/agent/streaming/parser/strategies/json_tool_strategy.py +26 -0
  99. autobyteus/agent/streaming/parser/strategies/registry.py +28 -0
  100. autobyteus/agent/streaming/parser/strategies/sentinel_strategy.py +23 -0
  101. autobyteus/agent/streaming/parser/strategies/xml_tag_strategy.py +21 -0
  102. autobyteus/agent/streaming/parser/stream_scanner.py +167 -0
  103. autobyteus/agent/streaming/parser/streaming_parser.py +212 -0
  104. autobyteus/agent/streaming/parser/tool_call_parsing.py +4 -0
  105. autobyteus/agent/streaming/parser/tool_constants.py +7 -0
  106. autobyteus/agent/streaming/parser/tool_syntax_registry.py +4 -0
  107. autobyteus/agent/streaming/parser/xml_tool_parsing_state_registry.py +55 -0
  108. autobyteus/agent/streaming/parsing_streaming_response_handler.py +4 -0
  109. autobyteus/agent/streaming/pass_through_streaming_response_handler.py +4 -0
  110. autobyteus/agent/streaming/queue_streamer.py +3 -57
  111. autobyteus/agent/streaming/segments/__init__.py +5 -0
  112. autobyteus/agent/streaming/segments/segment_events.py +81 -0
  113. autobyteus/agent/streaming/stream_event_payloads.py +2 -223
  114. autobyteus/agent/streaming/stream_events.py +3 -140
  115. autobyteus/agent/streaming/streaming_handler_factory.py +4 -0
  116. autobyteus/agent/streaming/streaming_response_handler.py +4 -0
  117. autobyteus/agent/streaming/streams/__init__.py +5 -0
  118. autobyteus/agent/streaming/streams/agent_event_stream.py +197 -0
  119. autobyteus/agent/streaming/utils/__init__.py +5 -0
  120. autobyteus/agent/streaming/utils/queue_streamer.py +59 -0
  121. autobyteus/agent/system_prompt_processor/__init__.py +2 -0
  122. autobyteus/agent/system_prompt_processor/available_skills_processor.py +96 -0
  123. autobyteus/agent/system_prompt_processor/base_processor.py +1 -1
  124. autobyteus/agent/system_prompt_processor/processor_meta.py +15 -2
  125. autobyteus/agent/system_prompt_processor/tool_manifest_injector_processor.py +39 -58
  126. autobyteus/agent/token_budget.py +56 -0
  127. autobyteus/agent/tool_execution_result_processor/memory_ingest_tool_result_processor.py +29 -0
  128. autobyteus/agent/tool_invocation.py +16 -40
  129. autobyteus/agent/tool_invocation_preprocessor/__init__.py +9 -0
  130. autobyteus/agent/tool_invocation_preprocessor/base_preprocessor.py +45 -0
  131. autobyteus/agent/tool_invocation_preprocessor/processor_definition.py +15 -0
  132. autobyteus/agent/tool_invocation_preprocessor/processor_meta.py +33 -0
  133. autobyteus/agent/tool_invocation_preprocessor/processor_registry.py +60 -0
  134. autobyteus/agent/utils/wait_for_idle.py +12 -14
  135. autobyteus/agent/workspace/base_workspace.py +6 -27
  136. autobyteus/agent_team/agent_team.py +3 -3
  137. autobyteus/agent_team/agent_team_builder.py +1 -41
  138. autobyteus/agent_team/bootstrap_steps/__init__.py +0 -4
  139. autobyteus/agent_team/bootstrap_steps/agent_configuration_preparation_step.py +8 -18
  140. autobyteus/agent_team/bootstrap_steps/agent_team_bootstrapper.py +4 -16
  141. autobyteus/agent_team/bootstrap_steps/base_agent_team_bootstrap_step.py +1 -2
  142. autobyteus/agent_team/bootstrap_steps/coordinator_initialization_step.py +1 -2
  143. autobyteus/agent_team/bootstrap_steps/task_notifier_initialization_step.py +1 -2
  144. autobyteus/agent_team/bootstrap_steps/team_context_initialization_step.py +4 -4
  145. autobyteus/agent_team/context/agent_team_config.py +6 -3
  146. autobyteus/agent_team/context/agent_team_context.py +25 -3
  147. autobyteus/agent_team/context/agent_team_runtime_state.py +9 -6
  148. autobyteus/agent_team/events/__init__.py +11 -0
  149. autobyteus/agent_team/events/agent_team_event_dispatcher.py +22 -9
  150. autobyteus/agent_team/events/agent_team_events.py +16 -0
  151. autobyteus/agent_team/events/event_store.py +57 -0
  152. autobyteus/agent_team/factory/agent_team_factory.py +8 -0
  153. autobyteus/agent_team/handlers/inter_agent_message_request_event_handler.py +18 -2
  154. autobyteus/agent_team/handlers/lifecycle_agent_team_event_handler.py +21 -5
  155. autobyteus/agent_team/handlers/process_user_message_event_handler.py +17 -8
  156. autobyteus/agent_team/handlers/tool_approval_team_event_handler.py +19 -4
  157. autobyteus/agent_team/runtime/agent_team_runtime.py +41 -10
  158. autobyteus/agent_team/runtime/agent_team_worker.py +69 -5
  159. autobyteus/agent_team/status/__init__.py +14 -0
  160. autobyteus/agent_team/status/agent_team_status.py +18 -0
  161. autobyteus/agent_team/status/agent_team_status_manager.py +33 -0
  162. autobyteus/agent_team/status/status_deriver.py +62 -0
  163. autobyteus/agent_team/status/status_update_utils.py +42 -0
  164. autobyteus/agent_team/streaming/__init__.py +2 -2
  165. autobyteus/agent_team/streaming/agent_team_event_notifier.py +6 -6
  166. autobyteus/agent_team/streaming/agent_team_stream_event_payloads.py +4 -4
  167. autobyteus/agent_team/streaming/agent_team_stream_events.py +3 -3
  168. autobyteus/agent_team/system_prompt_processor/__init__.py +6 -0
  169. autobyteus/agent_team/system_prompt_processor/team_manifest_injector_processor.py +76 -0
  170. autobyteus/agent_team/task_notification/task_notification_mode.py +19 -0
  171. autobyteus/agent_team/utils/wait_for_idle.py +4 -4
  172. autobyteus/cli/agent_cli.py +18 -10
  173. autobyteus/cli/agent_team_tui/app.py +14 -11
  174. autobyteus/cli/agent_team_tui/state.py +13 -15
  175. autobyteus/cli/agent_team_tui/widgets/agent_list_sidebar.py +15 -15
  176. autobyteus/cli/agent_team_tui/widgets/focus_pane.py +143 -36
  177. autobyteus/cli/agent_team_tui/widgets/renderables.py +1 -1
  178. autobyteus/cli/agent_team_tui/widgets/shared.py +25 -25
  179. autobyteus/cli/cli_display.py +193 -44
  180. autobyteus/cli/workflow_tui/app.py +9 -10
  181. autobyteus/cli/workflow_tui/state.py +14 -16
  182. autobyteus/cli/workflow_tui/widgets/agent_list_sidebar.py +15 -15
  183. autobyteus/cli/workflow_tui/widgets/focus_pane.py +137 -35
  184. autobyteus/cli/workflow_tui/widgets/renderables.py +1 -1
  185. autobyteus/cli/workflow_tui/widgets/shared.py +25 -25
  186. autobyteus/clients/autobyteus_client.py +94 -1
  187. autobyteus/events/event_types.py +11 -18
  188. autobyteus/llm/api/autobyteus_llm.py +33 -29
  189. autobyteus/llm/api/claude_llm.py +142 -36
  190. autobyteus/llm/api/gemini_llm.py +163 -59
  191. autobyteus/llm/api/grok_llm.py +1 -1
  192. autobyteus/llm/api/minimax_llm.py +26 -0
  193. autobyteus/llm/api/mistral_llm.py +113 -87
  194. autobyteus/llm/api/ollama_llm.py +9 -42
  195. autobyteus/llm/api/openai_compatible_llm.py +127 -91
  196. autobyteus/llm/api/openai_llm.py +3 -3
  197. autobyteus/llm/api/openai_responses_llm.py +324 -0
  198. autobyteus/llm/api/zhipu_llm.py +21 -2
  199. autobyteus/llm/autobyteus_provider.py +70 -60
  200. autobyteus/llm/base_llm.py +85 -81
  201. autobyteus/llm/converters/__init__.py +14 -0
  202. autobyteus/llm/converters/anthropic_tool_call_converter.py +37 -0
  203. autobyteus/llm/converters/gemini_tool_call_converter.py +57 -0
  204. autobyteus/llm/converters/mistral_tool_call_converter.py +37 -0
  205. autobyteus/llm/converters/openai_tool_call_converter.py +38 -0
  206. autobyteus/llm/extensions/base_extension.py +6 -12
  207. autobyteus/llm/extensions/token_usage_tracking_extension.py +45 -18
  208. autobyteus/llm/llm_factory.py +282 -204
  209. autobyteus/llm/lmstudio_provider.py +60 -49
  210. autobyteus/llm/models.py +35 -2
  211. autobyteus/llm/ollama_provider.py +60 -49
  212. autobyteus/llm/ollama_provider_resolver.py +0 -1
  213. autobyteus/llm/prompt_renderers/__init__.py +19 -0
  214. autobyteus/llm/prompt_renderers/anthropic_prompt_renderer.py +104 -0
  215. autobyteus/llm/prompt_renderers/autobyteus_prompt_renderer.py +19 -0
  216. autobyteus/llm/prompt_renderers/base_prompt_renderer.py +10 -0
  217. autobyteus/llm/prompt_renderers/gemini_prompt_renderer.py +63 -0
  218. autobyteus/llm/prompt_renderers/mistral_prompt_renderer.py +87 -0
  219. autobyteus/llm/prompt_renderers/ollama_prompt_renderer.py +51 -0
  220. autobyteus/llm/prompt_renderers/openai_chat_renderer.py +97 -0
  221. autobyteus/llm/prompt_renderers/openai_responses_renderer.py +101 -0
  222. autobyteus/llm/providers.py +1 -3
  223. autobyteus/llm/token_counter/claude_token_counter.py +56 -25
  224. autobyteus/llm/token_counter/mistral_token_counter.py +12 -8
  225. autobyteus/llm/token_counter/openai_token_counter.py +24 -5
  226. autobyteus/llm/token_counter/token_counter_factory.py +12 -5
  227. autobyteus/llm/utils/llm_config.py +6 -12
  228. autobyteus/llm/utils/media_payload_formatter.py +27 -20
  229. autobyteus/llm/utils/messages.py +55 -3
  230. autobyteus/llm/utils/response_types.py +3 -0
  231. autobyteus/llm/utils/tool_call_delta.py +31 -0
  232. autobyteus/memory/__init__.py +32 -0
  233. autobyteus/memory/active_transcript.py +69 -0
  234. autobyteus/memory/compaction/__init__.py +9 -0
  235. autobyteus/memory/compaction/compaction_result.py +8 -0
  236. autobyteus/memory/compaction/compactor.py +89 -0
  237. autobyteus/memory/compaction/summarizer.py +11 -0
  238. autobyteus/memory/compaction_snapshot_builder.py +84 -0
  239. autobyteus/memory/memory_manager.py +183 -0
  240. autobyteus/memory/models/__init__.py +14 -0
  241. autobyteus/memory/models/episodic_item.py +41 -0
  242. autobyteus/memory/models/memory_types.py +7 -0
  243. autobyteus/memory/models/raw_trace_item.py +79 -0
  244. autobyteus/memory/models/semantic_item.py +41 -0
  245. autobyteus/memory/models/tool_interaction.py +20 -0
  246. autobyteus/memory/policies/__init__.py +5 -0
  247. autobyteus/memory/policies/compaction_policy.py +16 -0
  248. autobyteus/memory/retrieval/__init__.py +7 -0
  249. autobyteus/memory/retrieval/memory_bundle.py +11 -0
  250. autobyteus/memory/retrieval/retriever.py +13 -0
  251. autobyteus/memory/store/__init__.py +7 -0
  252. autobyteus/memory/store/base_store.py +14 -0
  253. autobyteus/memory/store/file_store.py +98 -0
  254. autobyteus/memory/tool_interaction_builder.py +46 -0
  255. autobyteus/memory/turn_tracker.py +9 -0
  256. autobyteus/multimedia/audio/api/autobyteus_audio_client.py +19 -5
  257. autobyteus/multimedia/audio/api/gemini_audio_client.py +108 -16
  258. autobyteus/multimedia/audio/audio_client_factory.py +47 -9
  259. autobyteus/multimedia/audio/audio_model.py +2 -1
  260. autobyteus/multimedia/image/api/autobyteus_image_client.py +19 -5
  261. autobyteus/multimedia/image/api/gemini_image_client.py +38 -17
  262. autobyteus/multimedia/image/api/openai_image_client.py +125 -43
  263. autobyteus/multimedia/image/autobyteus_image_provider.py +2 -1
  264. autobyteus/multimedia/image/image_client_factory.py +47 -15
  265. autobyteus/multimedia/image/image_model.py +5 -2
  266. autobyteus/multimedia/providers.py +3 -2
  267. autobyteus/skills/loader.py +71 -0
  268. autobyteus/skills/model.py +11 -0
  269. autobyteus/skills/registry.py +70 -0
  270. autobyteus/task_management/tools/todo_tools/add_todo.py +2 -2
  271. autobyteus/task_management/tools/todo_tools/create_todo_list.py +2 -2
  272. autobyteus/task_management/tools/todo_tools/update_todo_status.py +2 -2
  273. autobyteus/tools/__init__.py +34 -47
  274. autobyteus/tools/base_tool.py +7 -0
  275. autobyteus/tools/file/__init__.py +2 -6
  276. autobyteus/tools/file/patch_file.py +149 -0
  277. autobyteus/tools/file/read_file.py +36 -5
  278. autobyteus/tools/file/write_file.py +4 -1
  279. autobyteus/tools/functional_tool.py +43 -6
  280. autobyteus/tools/mcp/__init__.py +2 -0
  281. autobyteus/tools/mcp/config_service.py +5 -1
  282. autobyteus/tools/mcp/server/__init__.py +2 -0
  283. autobyteus/tools/mcp/server/http_managed_mcp_server.py +1 -1
  284. autobyteus/tools/mcp/server/websocket_managed_mcp_server.py +141 -0
  285. autobyteus/tools/mcp/server_instance_manager.py +8 -1
  286. autobyteus/tools/mcp/types.py +61 -0
  287. autobyteus/tools/multimedia/audio_tools.py +70 -17
  288. autobyteus/tools/multimedia/download_media_tool.py +18 -4
  289. autobyteus/tools/multimedia/image_tools.py +246 -62
  290. autobyteus/tools/operation_executor/journal_manager.py +107 -0
  291. autobyteus/tools/operation_executor/operation_event_buffer.py +57 -0
  292. autobyteus/tools/operation_executor/operation_event_producer.py +29 -0
  293. autobyteus/tools/operation_executor/operation_executor.py +58 -0
  294. autobyteus/tools/registry/tool_definition.py +43 -2
  295. autobyteus/tools/skill/load_skill.py +50 -0
  296. autobyteus/tools/terminal/__init__.py +45 -0
  297. autobyteus/tools/terminal/ansi_utils.py +32 -0
  298. autobyteus/tools/terminal/background_process_manager.py +233 -0
  299. autobyteus/tools/terminal/output_buffer.py +105 -0
  300. autobyteus/tools/terminal/prompt_detector.py +63 -0
  301. autobyteus/tools/terminal/pty_session.py +241 -0
  302. autobyteus/tools/terminal/session_factory.py +20 -0
  303. autobyteus/tools/terminal/terminal_session_manager.py +226 -0
  304. autobyteus/tools/terminal/tools/__init__.py +13 -0
  305. autobyteus/tools/terminal/tools/get_process_output.py +81 -0
  306. autobyteus/tools/terminal/tools/run_bash.py +109 -0
  307. autobyteus/tools/terminal/tools/start_background_process.py +104 -0
  308. autobyteus/tools/terminal/tools/stop_background_process.py +67 -0
  309. autobyteus/tools/terminal/types.py +54 -0
  310. autobyteus/tools/terminal/wsl_tmux_session.py +221 -0
  311. autobyteus/tools/terminal/wsl_utils.py +156 -0
  312. autobyteus/tools/transaction_management/backup_handler.py +48 -0
  313. autobyteus/tools/transaction_management/operation_lifecycle_manager.py +62 -0
  314. autobyteus/tools/usage/__init__.py +1 -2
  315. autobyteus/tools/usage/formatters/__init__.py +17 -1
  316. autobyteus/tools/usage/formatters/base_formatter.py +8 -0
  317. autobyteus/tools/usage/formatters/default_xml_schema_formatter.py +2 -2
  318. autobyteus/tools/usage/formatters/mistral_json_schema_formatter.py +18 -0
  319. autobyteus/tools/usage/formatters/patch_file_xml_example_formatter.py +64 -0
  320. autobyteus/tools/usage/formatters/patch_file_xml_schema_formatter.py +31 -0
  321. autobyteus/tools/usage/formatters/run_bash_xml_example_formatter.py +32 -0
  322. autobyteus/tools/usage/formatters/run_bash_xml_schema_formatter.py +36 -0
  323. autobyteus/tools/usage/formatters/write_file_xml_example_formatter.py +53 -0
  324. autobyteus/tools/usage/formatters/write_file_xml_schema_formatter.py +31 -0
  325. autobyteus/tools/usage/providers/tool_manifest_provider.py +10 -10
  326. autobyteus/tools/usage/registries/__init__.py +1 -3
  327. autobyteus/tools/usage/registries/tool_formatting_registry.py +115 -8
  328. autobyteus/tools/usage/tool_schema_provider.py +51 -0
  329. autobyteus/tools/web/__init__.py +4 -0
  330. autobyteus/tools/web/read_url_tool.py +80 -0
  331. autobyteus/utils/diff_utils.py +271 -0
  332. autobyteus/utils/download_utils.py +109 -0
  333. autobyteus/utils/file_utils.py +57 -2
  334. autobyteus/utils/gemini_helper.py +56 -0
  335. autobyteus/utils/gemini_model_mapping.py +71 -0
  336. autobyteus/utils/llm_output_formatter.py +75 -0
  337. autobyteus/utils/tool_call_format.py +36 -0
  338. autobyteus/workflow/agentic_workflow.py +3 -3
  339. autobyteus/workflow/bootstrap_steps/agent_tool_injection_step.py +2 -2
  340. autobyteus/workflow/bootstrap_steps/base_workflow_bootstrap_step.py +2 -2
  341. autobyteus/workflow/bootstrap_steps/coordinator_initialization_step.py +2 -2
  342. autobyteus/workflow/bootstrap_steps/coordinator_prompt_preparation_step.py +3 -9
  343. autobyteus/workflow/bootstrap_steps/workflow_bootstrapper.py +6 -6
  344. autobyteus/workflow/bootstrap_steps/workflow_runtime_queue_initialization_step.py +2 -2
  345. autobyteus/workflow/context/workflow_context.py +3 -3
  346. autobyteus/workflow/context/workflow_runtime_state.py +5 -5
  347. autobyteus/workflow/events/workflow_event_dispatcher.py +5 -5
  348. autobyteus/workflow/handlers/lifecycle_workflow_event_handler.py +3 -3
  349. autobyteus/workflow/handlers/process_user_message_event_handler.py +5 -5
  350. autobyteus/workflow/handlers/tool_approval_workflow_event_handler.py +2 -2
  351. autobyteus/workflow/runtime/workflow_runtime.py +8 -8
  352. autobyteus/workflow/runtime/workflow_worker.py +3 -3
  353. autobyteus/workflow/status/__init__.py +11 -0
  354. autobyteus/workflow/status/workflow_status.py +19 -0
  355. autobyteus/workflow/status/workflow_status_manager.py +48 -0
  356. autobyteus/workflow/streaming/__init__.py +2 -2
  357. autobyteus/workflow/streaming/workflow_event_notifier.py +7 -7
  358. autobyteus/workflow/streaming/workflow_stream_event_payloads.py +4 -4
  359. autobyteus/workflow/streaming/workflow_stream_events.py +3 -3
  360. autobyteus/workflow/utils/wait_for_idle.py +4 -4
  361. autobyteus-1.2.3.dist-info/METADATA +293 -0
  362. autobyteus-1.2.3.dist-info/RECORD +600 -0
  363. {autobyteus-1.2.1.dist-info → autobyteus-1.2.3.dist-info}/WHEEL +1 -1
  364. {autobyteus-1.2.1.dist-info → autobyteus-1.2.3.dist-info}/top_level.txt +0 -1
  365. autobyteus/agent/bootstrap_steps/agent_runtime_queue_initialization_step.py +0 -57
  366. autobyteus/agent/hooks/__init__.py +0 -16
  367. autobyteus/agent/hooks/base_phase_hook.py +0 -78
  368. autobyteus/agent/hooks/hook_definition.py +0 -36
  369. autobyteus/agent/hooks/hook_meta.py +0 -37
  370. autobyteus/agent/hooks/hook_registry.py +0 -106
  371. autobyteus/agent/llm_response_processor/provider_aware_tool_usage_processor.py +0 -103
  372. autobyteus/agent/phases/__init__.py +0 -18
  373. autobyteus/agent/phases/discover.py +0 -53
  374. autobyteus/agent/phases/manager.py +0 -265
  375. autobyteus/agent/phases/transition_decorator.py +0 -40
  376. autobyteus/agent/phases/transition_info.py +0 -33
  377. autobyteus/agent/remote_agent.py +0 -244
  378. autobyteus/agent/workspace/workspace_definition.py +0 -36
  379. autobyteus/agent/workspace/workspace_meta.py +0 -37
  380. autobyteus/agent/workspace/workspace_registry.py +0 -72
  381. autobyteus/agent_team/bootstrap_steps/agent_team_runtime_queue_initialization_step.py +0 -25
  382. autobyteus/agent_team/bootstrap_steps/coordinator_prompt_preparation_step.py +0 -85
  383. autobyteus/agent_team/phases/__init__.py +0 -11
  384. autobyteus/agent_team/phases/agent_team_operational_phase.py +0 -19
  385. autobyteus/agent_team/phases/agent_team_phase_manager.py +0 -48
  386. autobyteus/llm/api/bedrock_llm.py +0 -92
  387. autobyteus/llm/api/groq_llm.py +0 -94
  388. autobyteus/llm/api/nvidia_llm.py +0 -108
  389. autobyteus/llm/utils/token_pricing_config.py +0 -87
  390. autobyteus/rpc/__init__.py +0 -73
  391. autobyteus/rpc/client/__init__.py +0 -17
  392. autobyteus/rpc/client/abstract_client_connection.py +0 -124
  393. autobyteus/rpc/client/client_connection_manager.py +0 -153
  394. autobyteus/rpc/client/sse_client_connection.py +0 -306
  395. autobyteus/rpc/client/stdio_client_connection.py +0 -280
  396. autobyteus/rpc/config/__init__.py +0 -13
  397. autobyteus/rpc/config/agent_server_config.py +0 -153
  398. autobyteus/rpc/config/agent_server_registry.py +0 -152
  399. autobyteus/rpc/hosting.py +0 -244
  400. autobyteus/rpc/protocol.py +0 -244
  401. autobyteus/rpc/server/__init__.py +0 -20
  402. autobyteus/rpc/server/agent_server_endpoint.py +0 -181
  403. autobyteus/rpc/server/base_method_handler.py +0 -40
  404. autobyteus/rpc/server/method_handlers.py +0 -259
  405. autobyteus/rpc/server/sse_server_handler.py +0 -182
  406. autobyteus/rpc/server/stdio_server_handler.py +0 -151
  407. autobyteus/rpc/server_main.py +0 -198
  408. autobyteus/rpc/transport_type.py +0 -13
  409. autobyteus/tools/bash/__init__.py +0 -2
  410. autobyteus/tools/bash/bash_executor.py +0 -100
  411. autobyteus/tools/browser/__init__.py +0 -2
  412. autobyteus/tools/browser/session_aware/browser_session_aware_navigate_to.py +0 -75
  413. autobyteus/tools/browser/session_aware/browser_session_aware_tool.py +0 -30
  414. autobyteus/tools/browser/session_aware/browser_session_aware_web_element_trigger.py +0 -154
  415. autobyteus/tools/browser/session_aware/browser_session_aware_webpage_reader.py +0 -89
  416. autobyteus/tools/browser/session_aware/browser_session_aware_webpage_screenshot_taker.py +0 -107
  417. autobyteus/tools/browser/session_aware/factory/browser_session_aware_web_element_trigger_factory.py +0 -14
  418. autobyteus/tools/browser/session_aware/factory/browser_session_aware_webpage_reader_factory.py +0 -26
  419. autobyteus/tools/browser/session_aware/factory/browser_session_aware_webpage_screenshot_taker_factory.py +0 -14
  420. autobyteus/tools/browser/session_aware/shared_browser_session.py +0 -11
  421. autobyteus/tools/browser/session_aware/shared_browser_session_manager.py +0 -25
  422. autobyteus/tools/browser/session_aware/web_element_action.py +0 -20
  423. autobyteus/tools/browser/standalone/__init__.py +0 -6
  424. autobyteus/tools/browser/standalone/factory/__init__.py +0 -0
  425. autobyteus/tools/browser/standalone/factory/webpage_reader_factory.py +0 -25
  426. autobyteus/tools/browser/standalone/factory/webpage_screenshot_taker_factory.py +0 -14
  427. autobyteus/tools/browser/standalone/navigate_to.py +0 -84
  428. autobyteus/tools/browser/standalone/web_page_pdf_generator.py +0 -101
  429. autobyteus/tools/browser/standalone/webpage_image_downloader.py +0 -169
  430. autobyteus/tools/browser/standalone/webpage_reader.py +0 -105
  431. autobyteus/tools/browser/standalone/webpage_screenshot_taker.py +0 -105
  432. autobyteus/tools/file/edit_file.py +0 -200
  433. autobyteus/tools/file/list_directory.py +0 -168
  434. autobyteus/tools/file/search_files.py +0 -188
  435. autobyteus/tools/timer.py +0 -175
  436. autobyteus/tools/usage/parsers/__init__.py +0 -22
  437. autobyteus/tools/usage/parsers/_json_extractor.py +0 -99
  438. autobyteus/tools/usage/parsers/_string_decoders.py +0 -18
  439. autobyteus/tools/usage/parsers/anthropic_xml_tool_usage_parser.py +0 -10
  440. autobyteus/tools/usage/parsers/base_parser.py +0 -41
  441. autobyteus/tools/usage/parsers/default_json_tool_usage_parser.py +0 -83
  442. autobyteus/tools/usage/parsers/default_xml_tool_usage_parser.py +0 -316
  443. autobyteus/tools/usage/parsers/exceptions.py +0 -13
  444. autobyteus/tools/usage/parsers/gemini_json_tool_usage_parser.py +0 -77
  445. autobyteus/tools/usage/parsers/openai_json_tool_usage_parser.py +0 -149
  446. autobyteus/tools/usage/parsers/provider_aware_tool_usage_parser.py +0 -59
  447. autobyteus/tools/usage/registries/tool_usage_parser_registry.py +0 -62
  448. autobyteus/workflow/phases/__init__.py +0 -11
  449. autobyteus/workflow/phases/workflow_operational_phase.py +0 -19
  450. autobyteus/workflow/phases/workflow_phase_manager.py +0 -48
  451. autobyteus-1.2.1.dist-info/METADATA +0 -205
  452. autobyteus-1.2.1.dist-info/RECORD +0 -511
  453. examples/__init__.py +0 -1
  454. examples/agent_team/__init__.py +0 -1
  455. examples/discover_phase_transitions.py +0 -104
  456. examples/run_agentic_software_engineer.py +0 -239
  457. examples/run_browser_agent.py +0 -262
  458. examples/run_google_slides_agent.py +0 -287
  459. examples/run_mcp_browser_client.py +0 -174
  460. examples/run_mcp_google_slides_client.py +0 -270
  461. examples/run_mcp_list_tools.py +0 -189
  462. examples/run_poem_writer.py +0 -284
  463. examples/run_sqlite_agent.py +0 -295
  464. /autobyteus/{tools/browser/session_aware → skills}/__init__.py +0 -0
  465. /autobyteus/tools/{browser/session_aware/factory → skill}/__init__.py +0 -0
  466. {autobyteus-1.2.1.dist-info → autobyteus-1.2.3.dist-info}/licenses/LICENSE +0 -0
@@ -13,13 +13,51 @@ from autobyteus.utils.parameter_schema import ParameterSchema, ParameterDefiniti
13
13
 
14
14
  logger = logging.getLogger(__name__)
15
15
 
16
- GEMINI_TTS_VOICES = [
17
- "Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus", "Aoede",
18
- "Callirrhoe", "Autonoe", "Enceladus", "Iapetus", "Umbriel", "Algieba",
19
- "Despina", "Erinome", "Algenib", "Rasalgethi", "Laomedeia", "Achernar",
20
- "Alnilam", "Schedar", "Gacrux", "Pulcherrima", "Achird", "Zubenelgenubi",
21
- "Vindemiatrix", "Sadachbia", "Sadaltager", "Sulafat"
16
+ # Enhanced metadata for Google Gemini TTS voices, including gender and description.
17
+ GEMINI_VOICE_DETAILS = {
18
+ "Zephyr": {"gender": "female", "description": "Bright, Higher pitch"},
19
+ "Puck": {"gender": "male", "description": "Upbeat, Middle pitch"},
20
+ "Charon": {"gender": "male", "description": "Informative, Lower pitch"},
21
+ "Kore": {"gender": "female", "description": "Firm, Middle pitch"},
22
+ "Fenrir": {"gender": "male", "description": "Excitable, Lower middle pitch"},
23
+ "Leda": {"gender": "female", "description": "Youthful, Higher pitch"},
24
+ "Orus": {"gender": "male", "description": "Firm, Lower middle pitch"},
25
+ "Aoede": {"gender": "female", "description": "Breezy, Middle pitch"},
26
+ "Callirrhoe": {"gender": "female", "description": "Easy-going, Middle pitch"},
27
+ "Autonoe": {"gender": "female", "description": "Bright, Middle pitch"},
28
+ "Enceladus": {"gender": "male", "description": "Breathy, Lower pitch"},
29
+ "Iapetus": {"gender": "male", "description": "Clear, Lower middle pitch"},
30
+ "Umbriel": {"gender": "male", "description": "Easy-going, Lower middle pitch"},
31
+ "Algieba": {"gender": "male", "description": "Smooth, Lower pitch"},
32
+ "Despina": {"gender": "female", "description": "Smooth, Middle pitch"},
33
+ "Erinome": {"gender": "female", "description": "Clear, Middle pitch"},
34
+ "Algenib": {"gender": "male", "description": "Gravelly, Lower pitch"},
35
+ "Rasalgethi": {"gender": "male", "description": "Informative, Middle pitch"},
36
+ "Laomedeia": {"gender": "female", "description": "Upbeat, Higher pitch"},
37
+ "Achernar": {"gender": "female", "description": "Soft, Higher pitch"},
38
+ "Alnilam": {"gender": "male", "description": "Firm, Lower middle pitch"},
39
+ "Schedar": {"gender": "male", "description": "Even, Lower middle pitch"},
40
+ "Gacrux": {"gender": "female", "description": "Mature, Middle pitch"},
41
+ "Pulcherrima": {"gender": "female", "description": "Forward, Middle pitch"},
42
+ "Achird": {"gender": "male", "description": "Friendly, Lower middle pitch"},
43
+ "Zubenelgenubi": {"gender": "male", "description": "Casual, Lower middle pitch"},
44
+ "Vindemiatrix": {"gender": "female", "description": "Gentle, Middle pitch"},
45
+ "Sadachbia": {"gender": "male", "description": "Lively, Lower pitch"},
46
+ "Sadaltager": {"gender": "male", "description": "Knowledgeable, Middle pitch"},
47
+ "Sulafat": {"gender": "female", "description": "Warm, Middle pitch"},
48
+ }
49
+
50
+ # The list of voice names, derived from the keys of the details dictionary.
51
+ # This is used for the `enum_values` to maintain compatibility.
52
+ GEMINI_TTS_VOICES = list(GEMINI_VOICE_DETAILS.keys())
53
+
54
+ # Generate a formatted string of voice metadata to be appended to parameter descriptions.
55
+ _voice_descriptions_list = [
56
+ f"- {name} ({details['gender']}): {details['description']}"
57
+ for name, details in GEMINI_VOICE_DETAILS.items()
22
58
  ]
59
+ GEMINI_VOICE_METADATA_DESC = "\n\nDetailed Voice Options:\n" + "\n".join(_voice_descriptions_list)
60
+
23
61
 
24
62
  OPENAI_TTS_VOICES = [
25
63
  "alloy", "ash", "ballad", "coral", "echo", "fable", "onyx",
@@ -64,7 +102,7 @@ class AudioClientFactory(metaclass=SingletonMeta):
64
102
  ParameterDefinition(
65
103
  name="voice",
66
104
  param_type=ParameterType.ENUM,
67
- description="The voice to assign to this speaker.",
105
+ description="The voice to assign to this speaker." + GEMINI_VOICE_METADATA_DESC,
68
106
  enum_values=GEMINI_TTS_VOICES,
69
107
  required=True
70
108
  )
@@ -84,7 +122,7 @@ class AudioClientFactory(metaclass=SingletonMeta):
84
122
  param_type=ParameterType.ENUM,
85
123
  default_value="Kore",
86
124
  enum_values=GEMINI_TTS_VOICES,
87
- description="The voice to use for single-speaker generation."
125
+ description="The voice to use for single-speaker generation." + GEMINI_VOICE_METADATA_DESC
88
126
  ),
89
127
  ParameterDefinition(
90
128
  name="style_instructions",
@@ -102,7 +140,7 @@ class AudioClientFactory(metaclass=SingletonMeta):
102
140
  gemini_tts_model = AudioModel(
103
141
  name="gemini-2.5-flash-tts",
104
142
  value="gemini-2.5-flash-preview-tts",
105
- provider=MultimediaProvider.GOOGLE,
143
+ provider=MultimediaProvider.GEMINI,
106
144
  client_class=GeminiAudioClient,
107
145
  parameter_schema=gemini_tts_schema
108
146
  )
@@ -79,7 +79,8 @@ class AudioModel(metaclass=AudioModelMeta):
79
79
  """Returns the unique identifier for the model."""
80
80
  if self.runtime == MultimediaRuntime.AUTOBYTEUS and self.host_url:
81
81
  try:
82
- host = urlparse(self.host_url).hostname
82
+ parsed = urlparse(self.host_url)
83
+ host = parsed.netloc or parsed.hostname or self.host_url
83
84
  return f"{self.name}@{host}"
84
85
  except Exception:
85
86
  return f"{self.name}@{self.host_url}" # Fallback
@@ -1,4 +1,5 @@
1
1
  import logging
2
+ import uuid
2
3
  from typing import Optional, List, Dict, Any, TYPE_CHECKING
3
4
  from autobyteus.clients import AutobyteusClient
4
5
  from autobyteus.multimedia.image.base_image_client import BaseImageClient
@@ -13,6 +14,7 @@ logger = logging.getLogger(__name__)
13
14
  class AutobyteusImageClient(BaseImageClient):
14
15
  """
15
16
  An image client that connects to an Autobyteus LLM server instance for image tasks.
17
+ Maintains a persistent session ID for stateful interactions (e.g. conversational editing).
16
18
  """
17
19
 
18
20
  def __init__(self, model: "ImageModel", config: "MultimediaConfig"):
@@ -21,7 +23,9 @@ class AutobyteusImageClient(BaseImageClient):
21
23
  raise ValueError("AutobyteusImageClient requires a host_url in its ImageModel.")
22
24
 
23
25
  self.autobyteus_client = AutobyteusClient(server_url=model.host_url)
24
- logger.info(f"AutobyteusImageClient initialized for model '{self.model.name}' on host '{model.host_url}'.")
26
+ self.session_id = str(uuid.uuid4())
27
+ logger.info(f"AutobyteusImageClient initialized for model '{self.model.name}' "
28
+ f"on host '{model.host_url}' with session_id '{self.session_id}'.")
25
29
 
26
30
  async def generate_image(
27
31
  self,
@@ -72,7 +76,7 @@ class AutobyteusImageClient(BaseImageClient):
72
76
  ) -> ImageGenerationResponse:
73
77
  """Internal helper to call the remote server."""
74
78
  try:
75
- logger.info(f"Sending image generation request for model '{self.model.name}' to {self.model.host_url}")
79
+ logger.info(f"Sending image generation request for model '{self.model.name}' to {self.model.host_url} (Session: {self.session_id})")
76
80
 
77
81
  # The model name for the remote server is the `value`, not the unique `model_identifier`
78
82
  model_name_for_server = self.model.name
@@ -84,7 +88,8 @@ class AutobyteusImageClient(BaseImageClient):
84
88
  prompt=prompt,
85
89
  input_image_urls=input_image_urls,
86
90
  mask_url=mask_url,
87
- generation_config=generation_config
91
+ generation_config=generation_config,
92
+ session_id=self.session_id
88
93
  )
89
94
 
90
95
  image_urls = response_data.get("image_urls", [])
@@ -98,7 +103,16 @@ class AutobyteusImageClient(BaseImageClient):
98
103
  raise
99
104
 
100
105
  async def cleanup(self):
101
- """Closes the underlying AutobyteusClient."""
106
+ """
107
+ Notifies the server to cleanup the session, then closes the underlying HTTP client.
108
+ """
102
109
  if self.autobyteus_client:
103
- await self.autobyteus_client.close()
110
+ try:
111
+ logger.info(f"Notifying server to cleanup image session '{self.session_id}'...")
112
+ await self.autobyteus_client.cleanup_image_session(self.session_id)
113
+ except Exception as e:
114
+ logger.error(f"Failed to cleanup remote image session '{self.session_id}': {e}")
115
+ finally:
116
+ await self.autobyteus_client.close()
117
+
104
118
  logger.debug("AutobyteusImageClient cleaned up.")
@@ -1,14 +1,13 @@
1
1
  import logging
2
2
  import base64
3
- import os
4
3
  from typing import Optional, List, Dict, Any, TYPE_CHECKING
5
- from google import genai
6
- from PIL import Image
7
- import requests
4
+ from google.genai import types as genai_types
8
5
 
9
6
  from autobyteus.multimedia.image.base_image_client import BaseImageClient
10
7
  from autobyteus.multimedia.utils.response_types import ImageGenerationResponse
11
8
  from autobyteus.multimedia.utils.api_utils import load_image_from_url
9
+ from autobyteus.utils.gemini_helper import initialize_gemini_client_with_runtime
10
+ from autobyteus.utils.gemini_model_mapping import resolve_model_for_runtime
12
11
 
13
12
  if TYPE_CHECKING:
14
13
  from autobyteus.multimedia.image.image_model import ImageModel
@@ -21,17 +20,15 @@ class GeminiImageClient(BaseImageClient):
21
20
  An image client that uses Google's Gemini models for image generation tasks.
22
21
 
23
22
  **Setup Requirements:**
24
- 1. **Authentication:** Set the `GEMINI_API_KEY` environment variable with your API key.
23
+ 1. **AI Studio Mode:** Set `GEMINI_API_KEY`.
24
+ 2. **Vertex AI Mode:** Set `VERTEX_AI_PROJECT` and `VERTEX_AI_LOCATION`.
25
25
  """
26
26
 
27
27
  def __init__(self, model: "ImageModel", config: "MultimediaConfig"):
28
28
  super().__init__(model, config)
29
- api_key = os.getenv("GEMINI_API_KEY")
30
- if not api_key:
31
- raise ValueError("Please set the GEMINI_API_KEY environment variable.")
32
29
 
33
30
  try:
34
- self.client = genai.Client()
31
+ self.client, self.runtime_info = initialize_gemini_client_with_runtime()
35
32
  self.async_client = self.client.aio
36
33
  logger.info(f"GeminiImageClient initialized for model '{self.model.name}'.")
37
34
  except Exception as e:
@@ -60,16 +57,40 @@ class GeminiImageClient(BaseImageClient):
60
57
  except Exception as e:
61
58
  logger.error(f"Skipping image at '{url}' due to loading error: {e}")
62
59
 
63
- # Note: The google-genai library uses the synchronous client for the `.generate_content` method on a model
64
- # even in an async context, as there isn't a direct async equivalent exposed for this specific call on the model object.
65
- # We use the top-level async client for other potential future calls if the library API changes.
66
- model_instance = self.client.get_generative_model(model_name=f"models/{self.model.value}")
67
- response = await model_instance.generate_content_async(contents=content)
60
+ config_dict: Dict[str, Any] = {}
61
+ if self.config and self.config.params:
62
+ config_dict.update(self.config.params)
63
+ if generation_config:
64
+ config_dict.update(generation_config)
65
+ if "response_modalities" not in config_dict:
66
+ if getattr(self, "runtime_info", None) and self.runtime_info.runtime == "vertex":
67
+ config_dict["response_modalities"] = ["TEXT", "IMAGE"]
68
+ else:
69
+ config_dict["response_modalities"] = ["IMAGE"]
70
+ config = genai_types.GenerateContentConfig(**config_dict)
71
+
72
+ # FIX: Removed 'models/' prefix from model_name to support Vertex AI
73
+ runtime_adjusted_model = resolve_model_for_runtime(
74
+ self.model.value,
75
+ modality="image",
76
+ runtime=getattr(self, "runtime_info", None) and self.runtime_info.runtime,
77
+ )
78
+ if runtime_adjusted_model != self.model.value:
79
+ logger.info(
80
+ "Using runtime-adjusted Gemini image model '%s' (requested '%s').",
81
+ runtime_adjusted_model,
82
+ self.model.value,
83
+ )
84
+ response = await self.async_client.models.generate_content(
85
+ model=runtime_adjusted_model,
86
+ contents=content,
87
+ config=config,
88
+ )
68
89
 
69
90
 
70
91
  image_urls = []
71
- for part in response.parts:
72
- if part.inline_data and "image" in part.inline_data.mime_type:
92
+ for part in response.parts or []:
93
+ if part.inline_data and part.inline_data.mime_type and "image" in part.inline_data.mime_type:
73
94
  image_bytes = part.inline_data.data
74
95
  base64_image = base64.b64encode(image_bytes).decode("utf-8")
75
96
  data_uri = f"data:{part.inline_data.mime_type};base64,{base64_image}"
@@ -77,7 +98,7 @@ class GeminiImageClient(BaseImageClient):
77
98
 
78
99
  if not image_urls:
79
100
  # Check for a safety-related refusal to generate content
80
- if response.prompt_feedback.block_reason:
101
+ if response.prompt_feedback and response.prompt_feedback.block_reason:
81
102
  reason = response.prompt_feedback.block_reason.name
82
103
  logger.error(f"Image generation blocked due to safety settings. Reason: {reason}")
83
104
  raise ValueError(f"Image generation failed due to safety settings: {reason}")
@@ -1,9 +1,14 @@
1
1
  import logging
2
2
  import os
3
+ import tempfile
4
+ from pathlib import Path
3
5
  from typing import Optional, List, Dict, Any, TYPE_CHECKING
6
+
4
7
  from openai import OpenAI
8
+
5
9
  from autobyteus.multimedia.image.base_image_client import BaseImageClient
6
10
  from autobyteus.multimedia.utils.response_types import ImageGenerationResponse
11
+ from autobyteus.utils.download_utils import download_file_from_url
7
12
 
8
13
  if TYPE_CHECKING:
9
14
  from autobyteus.multimedia.image.image_model import ImageModel
@@ -11,9 +16,19 @@ if TYPE_CHECKING:
11
16
 
12
17
  logger = logging.getLogger(__name__)
13
18
 
19
+
20
+ def _mime_type_from_format(output_format: str) -> str:
21
+ fmt = (output_format or "png").lower()
22
+ if fmt in {"jpg", "jpeg"}:
23
+ return "image/jpeg"
24
+ if fmt == "webp":
25
+ return "image/webp"
26
+ return "image/png"
27
+
28
+
14
29
  class OpenAIImageClient(BaseImageClient):
15
30
  """
16
- An image client that uses OpenAI's DALL-E models.
31
+ An image client that uses OpenAI's gpt-image series via the images API.
17
32
  """
18
33
 
19
34
  def __init__(self, model: "ImageModel", config: "MultimediaConfig"):
@@ -34,49 +49,68 @@ class OpenAIImageClient(BaseImageClient):
34
49
  **kwargs
35
50
  ) -> ImageGenerationResponse:
36
51
  """
37
- Generates an image using an OpenAI DALL-E model via the v1/images/generations endpoint.
38
- Note: This endpoint does not support image inputs, even for multimodal models like gpt-image-1.
52
+ Generates an image using OpenAI's images generation endpoint.
53
+ Note: This endpoint does not support image inputs.
39
54
  """
40
55
  if input_image_urls:
41
56
  logger.warning(
42
- f"The OpenAI `images.generate` API used by this client does not support input images. "
43
- f"The images provided for model '{self.model.value}' will be ignored. "
44
- f"To use image inputs, a client based on the Chat Completions API is required."
57
+ "The OpenAI `images.generate` API used by this client does not support input images. "
58
+ "The images provided for model '%s' will be ignored. "
59
+ "To use image inputs, a client based on the Chat Completions API is required.",
60
+ self.model.value,
45
61
  )
46
62
 
47
63
  try:
48
64
  image_model = self.model.value
49
- logger.info(f"Generating image with OpenAI model '{image_model}' and prompt: '{prompt[:50]}...'")
65
+ logger.info("Generating image with OpenAI model '%s' and prompt: '%s...'", image_model, prompt[:50])
50
66
 
51
67
  # Combine default config with any overrides
52
68
  final_config = self.config.to_dict().copy()
53
69
  if generation_config:
54
70
  final_config.update(generation_config)
55
-
56
- response = self.client.images.generate(
57
- model=image_model,
58
- prompt=prompt,
59
- n=final_config.get("n", 1),
60
- size=final_config.get("size", "1024x1024"),
61
- quality=final_config.get("quality", "standard"),
62
- style=final_config.get("style", "vivid"),
63
- response_format="url"
71
+ # Always request a single image for simplicity
72
+ final_config["n"] = 1
73
+
74
+ request_kwargs = {
75
+ "model": image_model,
76
+ "prompt": prompt,
77
+ "n": 1,
78
+ "size": final_config.get("size", "1024x1024"),
79
+ "quality": final_config.get("quality", "standard"),
80
+ }
81
+ if "output_format" in final_config:
82
+ request_kwargs["output_format"] = final_config["output_format"]
83
+ if "output_compression" in final_config:
84
+ request_kwargs["output_compression"] = final_config["output_compression"]
85
+
86
+ response = self.client.images.generate(**request_kwargs)
87
+
88
+ output_format = final_config.get("output_format", "png")
89
+ mime_type = _mime_type_from_format(output_format)
90
+ image_urls_list: List[str] = []
91
+ for img in response.data:
92
+ if getattr(img, "url", None):
93
+ image_urls_list.append(img.url)
94
+ elif getattr(img, "b64_json", None):
95
+ image_urls_list.append(f"data:{mime_type};base64,{img.b64_json}")
96
+
97
+ revised_prompt: Optional[str] = (
98
+ response.data[0].revised_prompt
99
+ if response.data and hasattr(response.data[0], "revised_prompt")
100
+ else None
64
101
  )
65
102
 
66
- image_urls_list: List[str] = [img.url for img in response.data if img.url]
67
- revised_prompt: Optional[str] = response.data[0].revised_prompt if response.data and hasattr(response.data[0], 'revised_prompt') else None
68
-
69
103
  if not image_urls_list:
70
- raise ValueError("OpenAI API did not return any image URLs.")
104
+ raise ValueError("OpenAI API did not return any image data.")
71
105
 
72
- logger.info(f"Successfully generated {len(image_urls_list)} image(s).")
106
+ logger.info("Successfully generated %s image(s).", len(image_urls_list))
73
107
 
74
108
  return ImageGenerationResponse(
75
109
  image_urls=image_urls_list,
76
110
  revised_prompt=revised_prompt
77
111
  )
78
112
  except Exception as e:
79
- logger.error(f"Error during OpenAI image generation: {str(e)}")
113
+ logger.error("Error during OpenAI image generation: %s", str(e))
80
114
  raise ValueError(f"OpenAI image generation failed: {str(e)}")
81
115
 
82
116
  async def edit_image(
@@ -95,49 +129,97 @@ class OpenAIImageClient(BaseImageClient):
95
129
 
96
130
  source_image_url = input_image_urls[0]
97
131
  if len(input_image_urls) > 1:
98
- logger.warning(f"OpenAI edit endpoint only supports one input image. Using '{source_image_url}' and ignoring the rest.")
132
+ logger.warning(
133
+ "OpenAI edit endpoint only supports one input image. Using '%s' and ignoring the rest.",
134
+ source_image_url,
135
+ )
99
136
 
137
+ temp_image_path: Optional[Path] = None
138
+ temp_mask_path: Optional[Path] = None
100
139
  try:
101
- logger.info(f"Editing image '{source_image_url}' with prompt: '{prompt[:50]}...'")
140
+ logger.info("Editing image '%s' with prompt: '%s...'", source_image_url, prompt[:50])
102
141
 
103
142
  # Combine default config with any overrides
104
143
  final_config = self.config.to_dict().copy()
105
144
  if generation_config:
106
145
  final_config.update(generation_config)
107
-
108
- with open(source_image_url, "rb") as image_file:
109
- mask_file = open(mask_url, "rb") if mask_url else None
146
+ # Always request a single edited image
147
+ final_config["n"] = 1
148
+
149
+ source_path = Path(source_image_url)
150
+ if not source_path.exists():
151
+ temp_image_file = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
152
+ temp_image_file.close()
153
+ temp_image_path = Path(temp_image_file.name)
154
+ await download_file_from_url(source_image_url, temp_image_path)
155
+ source_path = temp_image_path
156
+
157
+ if mask_url:
158
+ mask_path = Path(mask_url)
159
+ if not mask_path.exists():
160
+ temp_mask_file = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
161
+ temp_mask_file.close()
162
+ temp_mask_path = Path(temp_mask_file.name)
163
+ await download_file_from_url(mask_url, temp_mask_path)
164
+ mask_path = temp_mask_path
165
+ else:
166
+ mask_path = None
167
+
168
+ with open(source_path, "rb") as image_file:
169
+ mask_file = open(mask_path, "rb") if mask_path else None
110
170
  try:
111
- response = self.client.images.edit(
112
- image=image_file,
113
- mask=mask_file,
114
- prompt=prompt,
115
- model=self.model.value,
116
- n=final_config.get("n", 1),
117
- size=final_config.get("size", "1024x1024"),
118
- response_format="url"
119
- )
171
+ request_kwargs = {
172
+ "image": image_file,
173
+ "prompt": prompt,
174
+ "model": self.model.value,
175
+ "n": final_config.get("n", 1),
176
+ "size": final_config.get("size", "1024x1024"),
177
+ }
178
+ if mask_file:
179
+ request_kwargs["mask"] = mask_file
180
+ if "output_format" in final_config:
181
+ request_kwargs["output_format"] = final_config["output_format"]
182
+ if "output_compression" in final_config:
183
+ request_kwargs["output_compression"] = final_config["output_compression"]
184
+ response = self.client.images.edit(**request_kwargs)
120
185
  finally:
121
186
  if mask_file:
122
187
  mask_file.close()
123
188
 
124
- image_urls_list: List[str] = [img.url for img in response.data if img.url]
189
+ output_format = final_config.get("output_format", "png")
190
+ mime_type = _mime_type_from_format(output_format)
191
+ image_urls_list: List[str] = []
192
+ for img in response.data:
193
+ if getattr(img, "url", None):
194
+ image_urls_list.append(img.url)
195
+ elif getattr(img, "b64_json", None):
196
+ image_urls_list.append(f"data:{mime_type};base64,{img.b64_json}")
197
+
125
198
  if not image_urls_list:
126
- raise ValueError("OpenAI API did not return any edited image URLs.")
199
+ raise ValueError("OpenAI API did not return any edited image data.")
127
200
 
128
- logger.info(f"Successfully edited image, generated {len(image_urls_list)} version(s).")
201
+ logger.info("Successfully edited image, generated %s version(s).", len(image_urls_list))
129
202
  return ImageGenerationResponse(image_urls=image_urls_list)
130
203
 
131
204
  except FileNotFoundError as e:
132
- logger.error(f"Image file not found for editing: {e.filename}")
205
+ logger.error("Image file not found for editing: %s", e.filename)
133
206
  raise
134
207
  except Exception as e:
135
- logger.error(f"Error during OpenAI image editing: {str(e)}")
136
- # The API might return a 400 Bad Request if the model doesn't support edits
208
+ logger.error("Error during OpenAI image editing: %s", str(e))
137
209
  if "does not support image editing" in str(e):
138
210
  raise ValueError(f"The model '{self.model.value}' does not support the image editing endpoint.")
139
211
  raise ValueError(f"OpenAI image editing failed: {str(e)}")
140
-
212
+ finally:
213
+ if temp_image_path and temp_image_path.exists():
214
+ try:
215
+ temp_image_path.unlink()
216
+ except OSError:
217
+ logger.warning("Failed to clean up temp image file: %s", temp_image_path)
218
+ if temp_mask_path and temp_mask_path.exists():
219
+ try:
220
+ temp_mask_path.unlink()
221
+ except OSError:
222
+ logger.warning("Failed to clean up temp mask file: %s", temp_mask_path)
141
223
 
142
224
  async def cleanup(self):
143
225
  # The OpenAI client does not require explicit cleanup of a session.
@@ -86,7 +86,8 @@ class AutobyteusImageModelProvider:
86
86
  client_class=AutobyteusImageClient,
87
87
  runtime=MultimediaRuntime.AUTOBYTEUS,
88
88
  host_url=host_url,
89
- parameter_schema=model_info.get("parameter_schema")
89
+ parameter_schema=model_info.get("parameter_schema"),
90
+ description=model_info.get("description")
90
91
  )
91
92
 
92
93
  ImageClientFactory.register_model(image_model)
@@ -40,43 +40,75 @@ class ImageClientFactory(metaclass=SingletonMeta):
40
40
  """Initializes the registry with built-in image models and discovers remote ones."""
41
41
 
42
42
  # OpenAI Models
43
- gpt_image_1_schema = ParameterSchema(parameters=[
43
+ gpt_image_15_schema = ParameterSchema(parameters=[
44
44
  ParameterDefinition(name="n", param_type=ParameterType.INTEGER, default_value=1, enum_values=[1], description="The number of images to generate."),
45
45
  ParameterDefinition(name="size", param_type=ParameterType.ENUM, default_value="1024x1024", enum_values=["1024x1024", "1792x1024", "1024x1792"], description="The size of the generated images."),
46
- ParameterDefinition(name="quality", param_type=ParameterType.ENUM, default_value="hd", enum_values=["standard", "hd"], description="The quality of the image that will be generated."),
47
- ParameterDefinition(name="style", param_type=ParameterType.ENUM, default_value="vivid", enum_values=["vivid", "natural"], description="The style of the generated images.")
46
+ ParameterDefinition(name="quality", param_type=ParameterType.ENUM, default_value="auto", enum_values=["auto", "low", "medium", "high"], description="The quality of the image that will be generated.")
48
47
  ])
49
48
 
50
- gpt_image_1_model = ImageModel(
51
- name="gpt-image-1",
52
- value="gpt-image-1",
49
+ gemini_image_schema = ParameterSchema(parameters=[
50
+ ParameterDefinition(name="n", param_type=ParameterType.INTEGER, default_value=1, enum_values=[1], description="The number of images to generate."),
51
+ ParameterDefinition(name="size", param_type=ParameterType.ENUM, default_value="1024x1024", enum_values=["1024x1024", "1792x1024", "1024x1792"], description="The size of the generated images."),
52
+ ParameterDefinition(name="quality", param_type=ParameterType.ENUM, default_value="auto", enum_values=["auto", "low", "medium", "high"], description="The quality of the image that will be generated.")
53
+ ])
54
+
55
+ gpt_image_15_model = ImageModel(
56
+ name="gpt-image-1.5",
57
+ value="gpt-image-1.5",
53
58
  provider=MultimediaProvider.OPENAI,
54
59
  client_class=OpenAIImageClient,
55
- parameter_schema=gpt_image_1_schema
60
+ parameter_schema=gpt_image_15_schema,
61
+ description=(
62
+ "OpenAI's latest **stateless (single-turn)** image model with faster renders, improved text rendering, "
63
+ "and higher fidelity edits. Same API surface as gpt-image-1."
64
+ )
56
65
  )
57
66
 
58
67
  # Google Imagen Models (via Gemini API)
59
68
  imagen_model = ImageModel(
60
69
  name="imagen-4",
61
70
  value="imagen-4.0-generate-001",
62
- provider=MultimediaProvider.GOOGLE,
71
+ provider=MultimediaProvider.GEMINI,
63
72
  client_class=GeminiImageClient,
64
- parameter_schema=None # The genai library doesn't expose these as simple params
73
+ parameter_schema=None, # The genai library doesn't expose these as simple params
74
+ description=(
75
+ "A high-fidelity **stateless (single-turn)** model. "
76
+ "Does **NOT** support input images (text-to-image only). "
77
+ "Any provided input images will be ignored."
78
+ )
65
79
  )
66
80
 
67
- # Google Gemini Flash Image Model (aka "Nano Banana")
81
+ # Google Gemini 2.5 Flash Image (legacy, still widely available)
68
82
  gemini_flash_image_model = ImageModel(
69
- name="gemini-2.5-flash-image-preview",
70
- value="gemini-2.5-flash-image-preview",
71
- provider=MultimediaProvider.GOOGLE,
83
+ name="gemini-2.5-flash-image",
84
+ value="gemini-2.5-flash-image",
85
+ provider=MultimediaProvider.GEMINI,
86
+ client_class=GeminiImageClient,
87
+ parameter_schema=None, # Parameters handled by genai library
88
+ description=(
89
+ "Fast **conversational (multi-turn)** multimodal image model. "
90
+ "Supports context retention and input images for edits/variations."
91
+ )
92
+ )
93
+
94
+ # Google Gemini 3 Pro Image (aka "Nano Banana Pro")
95
+ gemini_pro_image_model = ImageModel(
96
+ name="gemini-3-pro-image-preview",
97
+ value="gemini-3-pro-image-preview",
98
+ provider=MultimediaProvider.GEMINI,
72
99
  client_class=GeminiImageClient,
73
- parameter_schema=None # Parameters are not exposed for this model via the genai library.
100
+ parameter_schema=None, # genai library handles options internally
101
+ description=(
102
+ "High-quality **conversational (multi-turn)** image model for complex edits and 4K renders. "
103
+ "Supports up to 14 reference images, advanced text rendering, and thinking mode."
104
+ )
74
105
  )
75
106
 
76
107
  models_to_register = [
77
- gpt_image_1_model,
108
+ gpt_image_15_model,
78
109
  imagen_model,
79
110
  gemini_flash_image_model,
111
+ gemini_pro_image_model,
80
112
  ]
81
113
 
82
114
  for model in models_to_register:
@@ -50,7 +50,8 @@ class ImageModel(metaclass=ImageModelMeta):
50
50
  client_class: Type["BaseImageClient"],
51
51
  parameter_schema: Optional[Union[Dict[str, Any], ParameterSchema]] = None,
52
52
  runtime: MultimediaRuntime = MultimediaRuntime.API,
53
- host_url: Optional[str] = None
53
+ host_url: Optional[str] = None,
54
+ description: Optional[str] = None
54
55
  ):
55
56
  self.name = name
56
57
  self.value = value
@@ -58,6 +59,7 @@ class ImageModel(metaclass=ImageModelMeta):
58
59
  self.client_class = client_class
59
60
  self.runtime = runtime
60
61
  self.host_url = host_url
62
+ self.description = description
61
63
 
62
64
  if isinstance(parameter_schema, dict):
63
65
  self.parameter_schema = ParameterSchema.from_dict(parameter_schema)
@@ -79,7 +81,8 @@ class ImageModel(metaclass=ImageModelMeta):
79
81
  """Returns the unique identifier for the model."""
80
82
  if self.runtime == MultimediaRuntime.AUTOBYTEUS and self.host_url:
81
83
  try:
82
- host = urlparse(self.host_url).hostname
84
+ parsed = urlparse(self.host_url)
85
+ host = parsed.netloc or parsed.hostname or self.host_url
83
86
  return f"{self.name}@{host}"
84
87
  except Exception:
85
88
  return f"{self.name}@{self.host_url}" # Fallback
@@ -2,5 +2,6 @@ from enum import Enum
2
2
 
3
3
  class MultimediaProvider(Enum):
4
4
  OPENAI = "OPENAI"
5
- GOOGLE = "GOOGLE"
6
- ALIBABA_QWEN = "ALIBABA_QWEN"
5
+ GEMINI = "GEMINI"
6
+ QWEN = "QWEN"
7
+ AUTOBYTEUS = "AUTOBYTEUS"