quantalogic 0.2.8__py3-none-any.whl → 0.2.10__py3-none-any.whl

quantalogic/agent.py

@@ -64,6 +64,7 @@ class Agent(BaseModel):
     event_emitter: EventEmitter = EventEmitter()
     config: AgentConfig
     task_to_solve: str
+    task_to_solve_summary: str = ""
     ask_for_user_validation: Callable[[str], bool] = console_ask_for_user_validation
     last_tool_call: dict[str, Any] = {}  # Stores the last tool call information
     total_tokens: int = 0  # Total tokens in the conversation
@@ -137,6 +138,9 @@ class Agent(BaseModel):
         logger.debug(f"Solving task... {task}")
         self._reset_session(task_to_solve=task, max_iterations=max_iterations)
 
+        # Generate task summary
+        self.task_to_solve_summary = self._generate_task_summary(task)
+
         # Add system prompt to memory
         self.memory.add(Message(role="system", content=self.config.system_prompt))
 
@@ -166,7 +170,7 @@ class Agent(BaseModel):
                 self._update_total_tokens(message_history=self.memory.memory, prompt=current_prompt)
 
                 # Emit event: Task Think Start after updating total tokens
-                self._emit_event("task_think_start")
+                self._emit_event("task_think_start", {"prompt": current_prompt})
 
                 self._compact_memory_if_needed(current_prompt)
 
@@ -443,6 +447,10 @@ class Agent(BaseModel):
             "You must analyze this answer and evaluate what to do next to solve the task.\n"
             "If the step failed, take a step back and rethink your approach.\n"
             "\n"
+            "--- Task to solve summary ---\n"
+            "\n"
+            f"{self.task_to_solve_summary}"
+            "\n"
             "--- Format ---\n"
             "\n"
             "You MUST respond with exactly two XML blocks formatted in markdown:\n"
@@ -546,6 +554,7 @@ class Agent(BaseModel):
             "\n### Tools:\n"
             "-----------------------------\n"
             f"{self._get_tools_names_prompt()}\n"
+            "\n"
             "### Variables:\n"
             "-----------------------------\n"
             f"{self._get_variable_prompt()}\n"
@@ -574,7 +583,7 @@ class Agent(BaseModel):
             "\n"
             "Available variables:\n"
             "\n"
-            f"{', '.join(self.variable_store.keys())}\n"
+            f"{', '.join(self.variable_store.keys())}\n" if len(self.variable_store.keys()) > 0 else "None\n"
         )
         return prompt_use_variables
 
@@ -619,6 +628,28 @@ class Agent(BaseModel):
         self.memory.memory = memory_copy
         return summary.response
 
+    def _generate_task_summary(self, content: str) -> str:
+        """Generate a concise summary of the given content using the generative model.
+        
+        Args:
+            content (str): The content to summarize
+            
+        Returns:
+            str: Generated summary
+        """
+        try:
+            prompt = (
+                "Rewrite this task in a precise, dense, and concise manner:\n"
+                f"{content}\n"
+                "Summary should be 2-3 sentences maximum. No extra comments should be added.\n"
+            )
+            result = self.model.generate(prompt=prompt)
+            logger.debug(f"Generated summary: {result.response}")
+            return result.response
+        except Exception as e:
+            logger.error(f"Error generating summary: {str(e)}")
+            return f"Summary generation failed: {str(e)}"
+
     def _update_session_memory(self, user_content: str, assistant_content: str) -> None:
         """
         Log session messages to memory and emit events.
@@ -639,3 +670,5 @@ class Agent(BaseModel):
             "session_add_message",
             {"role": "assistant", "content": assistant_content},
         )
+        
+

quantalogic/agent_config.py

@@ -13,6 +13,7 @@ from quantalogic.tools import (
     InputQuestionTool,
     ListDirectoryTool,
     LLMTool,
+    LLMVisionTool,
     MarkitdownTool,
     NodeJsTool,
     PythonTool,
@@ -28,109 +29,141 @@ from quantalogic.tools import (
 MODEL_NAME = "deepseek/deepseek-chat"
 
 
-def create_agent(model_name) -> Agent:
+def create_agent(model_name: str, vision_model_name: str | None) -> Agent:
     """Create an agent with the specified model and tools.
 
     Args:
         model_name (str): Name of the model to use
+        vision_model_name (str | None): Name of the vision model to use
+
+    Returns:
+        Agent: An agent with the specified model and tools
     """
+    tools = [
+        TaskCompleteTool(),
+        ReadFileTool(),
+        ReadFileBlockTool(),
+        WriteFileTool(),
+        EditWholeContentTool(),
+        InputQuestionTool(),
+        ListDirectoryTool(),
+        ExecuteBashCommandTool(),
+        ReplaceInFileTool(),
+        RipgrepTool(),
+        SearchDefinitionNames(),
+        MarkitdownTool(),
+        LLMTool(model_name=model_name),
+        DownloadHttpFileTool(),
+    ]
+
+    if vision_model_name:
+        tools.append(LLMVisionTool(model_name=vision_model_name))
+
     return Agent(
         model_name=model_name,
-        tools=[
-            TaskCompleteTool(),
-            ReadFileTool(),
-            ReadFileBlockTool(),
-            WriteFileTool(),
-            EditWholeContentTool(),
-            InputQuestionTool(),
-            ListDirectoryTool(),
-            ExecuteBashCommandTool(),
-            ReplaceInFileTool(),
-            RipgrepTool(),
-            SearchDefinitionNames(),
-            MarkitdownTool(),
-            LLMTool(model_name=model_name),
-            DownloadHttpFileTool(),
-        ],
+        tools=tools,
     )
 
 
-def create_interpreter_agent(model_name: str) -> Agent:
+def create_interpreter_agent(model_name: str, vision_model_name: str | None) -> Agent:
     """Create an interpreter agent with the specified model and tools.
 
     Args:
         model_name (str): Name of the model to use
-    """
-    return Agent(
-        model_name=model_name,
-        tools=[
-            TaskCompleteTool(),
-            ReadFileTool(),
-            ReadFileBlockTool(),
-            WriteFileTool(),
-            EditWholeContentTool(),
-            InputQuestionTool(),
-            ListDirectoryTool(),
-            ExecuteBashCommandTool(),
-            ReplaceInFileTool(),
-            RipgrepTool(),
-            PythonTool(),
-            NodeJsTool(),
-            SearchDefinitionNames(),
-            DownloadHttpFileTool(),
-        ],
-    )
-
+        vision_model_name (str | None): Name of the vision model to use
 
-def create_full_agent(model_name: str) -> Agent:
+    Returns:
+        Agent: An interpreter agent with the specified model and tools
+    """
+    tools = [
+        TaskCompleteTool(),
+        ReadFileTool(),
+        ReadFileBlockTool(),
+        WriteFileTool(),
+        EditWholeContentTool(),
+        InputQuestionTool(),
+        ListDirectoryTool(),
+        ExecuteBashCommandTool(),
+        ReplaceInFileTool(),
+        RipgrepTool(),
+        PythonTool(),
+        NodeJsTool(),
+        SearchDefinitionNames(),
+        MarkitdownTool(),
+        LLMTool(model_name=model_name),
+        DownloadHttpFileTool(),
+    ]
+    return Agent(model_name=model_name, tools=tools)
+
+
+def create_full_agent(model_name: str, vision_model_name: str | None) -> Agent:
     """Create an agent with the specified model and many tools.
 
     Args:
         model_name (str): Name of the model to use
+        vision_model_name (str | None): Name of the vision model to use
+
+    Returns:
+        Agent: An agent with the specified model and tools
+
     """
+    tools=[
+        TaskCompleteTool(),
+        ReadFileTool(),
+        ReadFileBlockTool(),
+        WriteFileTool(),
+        EditWholeContentTool(),
+        InputQuestionTool(),
+        ListDirectoryTool(),
+        ExecuteBashCommandTool(),
+        ReplaceInFileTool(),
+        RipgrepTool(),
+        PythonTool(),
+        NodeJsTool(),
+        SearchDefinitionNames(),
+        MarkitdownTool(),
+        LLMTool(model_name=model_name),
+        DownloadHttpFileTool(),
+    ]
+
+    if vision_model_name:
+        tools.append(LLMVisionTool(model_name=vision_model_name))
+
     return Agent(
         model_name=model_name,
-        tools=[
-            TaskCompleteTool(),
-            ReadFileTool(),
-            ReadFileBlockTool(),
-            WriteFileTool(),
-            EditWholeContentTool(),
-            InputQuestionTool(),
-            ListDirectoryTool(),
-            ExecuteBashCommandTool(),
-            ReplaceInFileTool(),
-            RipgrepTool(),
-            PythonTool(),
-            NodeJsTool(),
-            SearchDefinitionNames(),
-            MarkitdownTool(),
-            LLMTool(model_name=model_name),
-            DownloadHttpFileTool(),
-        ],
+        tools=tools,
     )
 
 
-def create_orchestrator_agent(model_name: str) -> Agent:
+def create_orchestrator_agent(model_name: str, vision_model_name: str | None = None) -> Agent:
     """Create an agent with the specified model and tools.
 
     Args:
         model_name (str): Name of the model to use
+        vision_model_name (str | None): Name of the vision model to use
+
+    Returns:
+        Agent: An agent with the specified model and tools
     """
     # Rebuild AgentTool to resolve forward references
     AgentTool.model_rebuild()
 
     coding_agent_instance = create_coding_agent(model_name)
 
+    tools = [
+        TaskCompleteTool(),
+        ListDirectoryTool(),
+        ReadFileBlockTool(),
+        RipgrepTool(),
+        SearchDefinitionNames(),
+        LLMTool(model_name=MODEL_NAME),
+        AgentTool(agent=coding_agent_instance, agent_role="software expert", name="coder_agent_tool"),
+    ]
+
+    if vision_model_name:
+        tools.append(LLMVisionTool(model_name=vision_model_name))
+
     return Agent(
         model_name=model_name,
-        tools=[
-            TaskCompleteTool(),
-            ListDirectoryTool(),
-            ReadFileBlockTool(),
-            RipgrepTool(),
-            SearchDefinitionNames(),
-            LLMTool(model_name=MODEL_NAME),
-            AgentTool(agent=coding_agent_instance, agent_role="software expert", name="coder_agent_tool"),
-        ],
+        tools=tools,
     )

quantalogic/coding_agent.py

@@ -5,6 +5,7 @@ from quantalogic.tools import (
     InputQuestionTool,
     ListDirectoryTool,
     LLMTool,
+    LLMVisionTool,
     ReadFileBlockTool,
     ReadFileTool,
     ReplaceInFileTool,
@@ -17,11 +18,12 @@ from quantalogic.utils import get_coding_environment
 from quantalogic.utils.get_quantalogic_rules_content import get_quantalogic_rules_file_content
 
 
-def create_coding_agent(model_name: str, basic: bool = False) -> Agent:
+def create_coding_agent(model_name: str,vision_model_name: str | None = None, basic: bool = False) -> Agent:
     """Creates and configures a coding agent with a comprehensive set of tools.
 
     Args:
         model_name (str): Name of the language model to use for the agent's core capabilities
+        vision_model_name (str | None): Name of the vision model to use for the agent's core capabilities
         basic (bool, optional): If True, the agent will be configured with a basic set of tools.
 
     Returns:
@@ -59,6 +61,9 @@ def create_coding_agent(model_name: str, basic: bool = False) -> Agent:
         InputQuestionTool(),
     ]
 
+    if vision_model_name:
+        tools.append(LLMVisionTool(model_name=vision_model_name))
+
     if not basic:
         tools.append(
             LLMTool(

quantalogic/generative_model.py

@@ -5,16 +5,17 @@ from litellm import completion, exceptions, get_max_tokens, get_model_info, toke
 from loguru import logger
 from pydantic import BaseModel, Field, field_validator
 
-MIN_RETRIES = 3
+MIN_RETRIES = 1
 
 
 class Message(BaseModel):
     """Represents a message in a conversation with a specific role and content."""
 
     role: str = Field(..., min_length=1)
-    content: str = Field(..., min_length=1)
+    content: str | dict = Field(..., min_length=1)
+    image_url: str | None = Field(default=None, pattern=r"^https?://")
 
-    @field_validator("role", "content")
+    @field_validator("role")
     @classmethod
     def validate_not_empty(cls, v: str) -> str:
         """Validate that the field is not empty or whitespace-only."""
@@ -22,6 +23,26 @@ class Message(BaseModel):
             raise ValueError("Field cannot be empty or whitespace-only")
         return v
 
+    @field_validator("content")
+    @classmethod
+    def validate_content(cls, v: str | dict) -> str | dict:
+        """Validate content based on its type."""
+        if isinstance(v, str):
+            if not v or not v.strip():
+                raise ValueError("Text content cannot be empty or whitespace-only")
+        elif isinstance(v, dict):
+            if not v.get("text") or not v.get("image_url"):
+                raise ValueError("Multimodal content must have both text and image_url")
+        return v
+
+    @field_validator("image_url")
+    @classmethod
+    def validate_image_url(cls, v: str | None) -> str | None:
+        """Validate image URL format if present."""
+        if v and not v.startswith(("http://", "https://")):
+            raise ValueError("Image URL must start with http:// or https://")
+        return v
+
 
 class TokenUsage(BaseModel):
     """Represents token usage statistics for a language model."""
@@ -59,6 +80,7 @@ class GenerativeModel:
             temperature: Sampling temperature between 0 and 1.
                 Defaults to 0.7.
         """
+        logger.debug(f"Initializing GenerativeModel with model={model}, temperature={temperature}")
         self.model = model
         self.temperature = temperature
 
@@ -85,15 +107,16 @@ class GenerativeModel:
     )
 
     # Retry on specific retriable exceptions
-    def generate_with_history(self, messages_history: list[Message], prompt: str) -> ResponseStats:
-        """Generate a response with conversation history.
+    def generate_with_history(self, messages_history: list[Message], prompt: str, image_url: str | None = None) -> ResponseStats:
+        """Generate a response with conversation history and optional image.
 
-        Generates a response based on previous conversation messages
-        and a new user prompt.
+        Generates a response based on previous conversation messages,
+        a new user prompt, and an optional image URL.
 
         Args:
             messages_history: Previous conversation messages.
             prompt: Current user prompt.
+            image_url: Optional image URL for visual queries.
 
         Returns:
             Detailed response statistics.
@@ -105,7 +128,22 @@ class GenerativeModel:
             Exception: For other unexpected errors.
         """
         messages = [{"role": msg.role, "content": str(msg.content)} for msg in messages_history]
-        messages.append({"role": "user", "content": str(prompt)})
+        
+        if image_url:
+            messages.append({
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": str(prompt)},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": image_url
+                        }
+                    }
+                ]
+            })
+        else:
+            messages.append({"role": "user", "content": str(prompt)})
 
         try:
             logger.debug(f"Generating response for prompt: {prompt}")
@@ -140,9 +178,12 @@ class GenerativeModel:
             }
 
             logger.error("LLM Generation Error: {}", error_details)
+            logger.debug(f"Error details: {error_details}")
+            logger.debug(f"Model: {self.model}, Temperature: {self.temperature}")
 
             # Handle authentication and permission errors
             if isinstance(e, self.AUTH_EXCEPTIONS):
+                logger.debug("Authentication error occurred")
                 raise openai.AuthenticationError(
                     f"Authentication failed with provider {error_details['provider']}"
                 ) from e
@@ -162,7 +203,7 @@ class GenerativeModel:
             # Wrap unknown errors in APIError
             raise openai.APIError(f"Unexpected error during generation: {str(e)}") from e
 
-    def generate(self, prompt: str) -> ResponseStats:
+    def generate(self, prompt: str, image_url: str | None = None) -> ResponseStats:
         """Generate a response without conversation history.
 
         Generates a response for a single user prompt without
@@ -170,11 +211,12 @@ class GenerativeModel:
 
         Args:
             prompt: User prompt.
+            image_url: Optional image URL for visual queries.
 
         Returns:
             Detailed response statistics.
         """
-        return self.generate_with_history([], prompt)
+        return self.generate_with_history([], prompt, image_url)
 
     def get_max_tokens(self) -> int:
         """Get the maximum number of tokens that can be generated by the model."""
@@ -182,8 +224,11 @@ class GenerativeModel:
 
     def token_counter(self, messages: list[Message]) -> int:
         """Count the number of tokens in a list of messages."""
+        logger.debug(f"Counting tokens for {len(messages)} messages using model {self.model}")
         litellm_messages = [{"role": msg.role, "content": str(msg.content)} for msg in messages]
-        return token_counter(model=self.model, messages=litellm_messages)
+        token_count = token_counter(model=self.model, messages=litellm_messages)
+        logger.debug(f"Token count: {token_count}")
+        return token_count
 
     def token_counter_with_history(self, messages_history: list[Message], prompt: str) -> int:
         """Count the number of tokens in a list of messages and a prompt."""
@@ -193,12 +238,18 @@ class GenerativeModel:
 
     def get_model_info(self) -> dict | None:
         """Get information about the model."""
+        logger.debug(f"Retrieving model info for {self.model}")
         model_info = get_model_info(self.model)
 
         if not model_info:
-            # Search without prefix "openrouter/"
+            logger.debug("Model info not found, trying without openrouter/ prefix")
             model_info = get_model_info(self.model.replace("openrouter/", ""))
 
+        if model_info:
+            logger.debug(f"Model info retrieved: {model_info.keys()}")
+        else:
+            logger.debug("No model info available")
+            
         return model_info
 
     def get_model_max_input_tokens(self) -> int:

quantalogic/main.py

@@ -33,20 +33,20 @@ from quantalogic.version import get_version  # noqa: E402
 AGENT_MODES = ["code", "basic", "interpreter", "full", "code-basic"]
 
 
-def create_agent_for_mode(mode: str, model_name: str) -> Agent:
+def create_agent_for_mode(mode: str, model_name: str, vision_model_name: str | None) -> Agent:
     """Create an agent based on the specified mode."""
     logger.debug(f"Creating agent for mode: {mode} with model: {model_name}")
     if mode == "code":
         logger.debug("Creating code agent without basic mode")
-        return create_coding_agent(model_name, basic=False)
+        return create_coding_agent(model_name, vision_model_name, basic=False)
     if mode == "code-basic":
-        return create_coding_agent(model_name, basic=True)
+        return create_coding_agent(model_name, vision_model_name, basic=True)
     elif mode == "basic":
-        return create_orchestrator_agent(model_name)
+        return create_orchestrator_agent(model_name, vision_model_name)
     elif mode == "full":
-        return create_full_agent(model_name)
+        return create_full_agent(model_name, vision_model_name)
     elif mode == "interpreter":
-        return create_interpreter_agent(model_name)
+        return create_interpreter_agent(model_name, vision_model_name)
     else:
         raise ValueError(f"Unknown agent mode: {mode}")
 
@@ -54,13 +54,18 @@ def create_agent_for_mode(mode: str, model_name: str) -> Agent:
 def configure_logger(log_level: str) -> None:
     """Configure the logger with the specified log level and format."""
     logger.remove()
-    logger.add(sys.stderr, level=log_level.upper(), format="{time} | {level} | {message}")
+    logger.add(
+        sys.stderr,
+        level=log_level.upper(),
+        format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> | <cyan>{process}</cyan> | <magenta>{file}:{line}</magenta> | {message}",
+    )
     logger.info(f"Log level set to: {log_level}")
 
 
 def set_litellm_verbose(verbose_mode: bool) -> None:
     """Set the verbosity of the litellm library."""
     import litellm
+
     litellm.set_verbose = verbose_mode
 
 
@@ -87,7 +92,7 @@ def get_task_from_file(file_path: str) -> str:
         raise Exception(f"Unexpected error reading file: {e}")
 
 
-def display_welcome_message(console: Console, model_name: str) -> None:
+def display_welcome_message(console: Console, model_name: str, vision_model_name: str | None) -> None:
     """Display the welcome message and instructions."""
     version = get_version()
     console.print(
@@ -97,9 +102,10 @@ def display_welcome_message(console: Console, model_name: str) -> None:
             "1. [bold]Describe your task[/bold]: Tell the AI what you need help with.\n"
             "2. [bold]Submit your task[/bold]: Press [bold]Enter[/bold] twice to send your request.\n\n"
             "3. [bold]Exit the app[/bold]: Leave the input blank and press [bold]Enter[/bold] twice to close the assistant.\n\n"
-            f"[yellow]ℹ️ System Info:[/yellow]\n\n"
-            f"- Version: {get_version()}\n"
-            f"- Model: {model_name}\n\n"
+            f"[yellow] 🤖 System Info:[/yellow]\n\n"
+            "\n"
+            f"- Model: {model_name}\n"
+            f"- Vision Model: {vision_model_name}\n\n"
             "[bold magenta]💡 Pro Tips:[/bold magenta]\n\n"
             "- Be as specific as possible in your task description to get the best results!\n"
             "- Use clear and concise language when describing your task\n"
@@ -118,18 +124,38 @@ def display_welcome_message(console: Console, model_name: str) -> None:
     default=MODEL_NAME,
     help='Specify the model to use (litellm format, e.g. "openrouter/deepseek-chat").',
 )
-@click.option("--log", type=click.Choice(["info", "debug", "warning"]), default="info", help="Set logging level (info/debug/warning).")
+@click.option(
+    "--log",
+    type=click.Choice(["info", "debug", "warning"]),
+    default="info",
+    help="Set logging level (info/debug/warning).",
+)
 @click.option("--verbose", is_flag=True, help="Enable verbose output.")
 @click.option("--mode", type=click.Choice(AGENT_MODES), default="code", help="Agent mode (code/search/full).")
+@click.option(
+    "--vision-model-name",
+    default=None,
+    help='Specify the vision model to use (litellm format, e.g. "openrouter/A/gpt-4o-mini").',
+)
 @click.pass_context
-def cli(ctx: click.Context, version: bool, model_name: str, verbose: bool, mode: str, log: str) -> None:
+def cli(
+    ctx: click.Context,
+    version: bool,
+    model_name: str,
+    verbose: bool,
+    mode: str,
+    log: str,
+    vision_model_name: str | None,
+) -> None:
     """QuantaLogic AI Assistant - A powerful AI tool for various tasks."""
     if version:
         console = Console()
         console.print(f"QuantaLogic version: {get_version()}")
         sys.exit(0)
     if ctx.invoked_subcommand is None:
-        ctx.invoke(task, model_name=model_name, verbose=verbose, mode=mode, log=log)
+        ctx.invoke(
+            task, model_name=model_name, verbose=verbose, mode=mode, log=log, vision_model_name=vision_model_name
+        )
 
 
 @cli.command()
@@ -141,9 +167,27 @@ def cli(ctx: click.Context, version: bool, model_name: str, verbose: bool, mode:
 )
 @click.option("--verbose", is_flag=True, help="Enable verbose output.")
 @click.option("--mode", type=click.Choice(AGENT_MODES), default="code", help="Agent mode (code/search/full).")
-@click.option("--log", type=click.Choice(["info", "debug", "warning"]), default="info", help="Set logging level (info/debug/warning).")
+@click.option(
+    "--log",
+    type=click.Choice(["info", "debug", "warning"]),
+    default="info",
+    help="Set logging level (info/debug/warning).",
+)
+@click.option(
+    "--vision-model-name",
+    default=None,
+    help='Specify the vision model to use (litellm format, e.g. "openrouter/openai/gpt-4o-mini").',
+)
 @click.argument("task", required=False)
-def task(file: Optional[str], model_name: str, verbose: bool, mode: str, log: str, task: Optional[str]) -> None:
+def task(
+    file: Optional[str],
+    model_name: str,
+    verbose: bool,
+    mode: str,
+    log: str,
+    vision_model_name: str | None,
+    task: Optional[str],
+) -> None:
     """Execute a task with the QuantaLogic AI Assistant."""
     console = Console()
     switch_verbose(verbose, log)
@@ -155,7 +199,7 @@ def task(file: Optional[str], model_name: str, verbose: bool, mode: str, log: st
             if task:
                 task_content = task
             else:
-                display_welcome_message(console, model_name)
+                display_welcome_message(console, model_name, vision_model_name)
                 logger.info("Waiting for user input...")
                 task_content = get_multiline_input(console).strip()
                 logger.info(f"User input received. Task content: {task_content}")
@@ -177,7 +221,7 @@ def task(file: Optional[str], model_name: str, verbose: bool, mode: str, log: st
                 sys.exit(0)
 
         logger.debug(f"Creating agent for mode: {mode} with model: {model_name}")
-        agent = create_agent_for_mode(mode, model_name)
+        agent = create_agent_for_mode(mode, model_name, vision_model_name=vision_model_name)
         logger.debug(f"Created agent for mode: {mode} with model: {model_name}")
 
         events = [

quantalogic/prompts.py

@@ -14,11 +14,10 @@ Tasks will be presented within XML tags:
 ### Response Protocol
 Every response must contain exactly two XML blocks:
 
-Be very concise and very precise in the <thinking> block
-
 1. Analysis Block:
 ```xml
 <thinking>
+  <!-- You must follow this precise format, be very concise and very precise -->
  <task_analysis_if_no_history> 
    Only if no conversation history:
     * Rewrite the <task> and its context with your own words in detailed, clear, and specific manner.
@@ -51,20 +50,24 @@ Be very concise and very precise in the <thinking> block
     </result>
   </last_observation>
   <progess_analysis>
+     <!-- if there is a conversation history -->
     * Detail each step failed and completed so far.
     * Identify and evaluate any blockers or challenges to the progress of global task.
+    * Identify repetitions: if repeated steps, take a step back and rethink your approach.
     * Provide potential solutions, and if needed, suggest reevaluating the approach and the plan.
   </progess_analysis>
   <variables>
+      <!-- if there is a conversation history -->
     * List all variable names and concisely describe their current values.
   </variables>
   <next_steps>
     * Outline immediate actions required.
     * Justify tool selection and parameter choices.
-    * Think about variable interpolation to minimize generation of tokens.
-    * Consider alternatives if previous attempts were unsuccessful.
+    * Prefer variable interpolation if possible, to minimize generation of tokens.
+    * Consider alternatives, take a step back if previous attempts were unsuccessful to review the plan.
   </next_steps>
   <taskpad>
+    <!-- optional -->
     <note>Use this to record notes about intermediate steps.</note>
   </taskpad>
 </thinking>
@@ -73,25 +76,15 @@ Be very concise and very precise in the <thinking> block
 2. Action Block:
 ```xml
 <tool_name>
-    <parameter1>value1</parameter1>
+    <!-- tool_name is the name of the tool from available tools -->
+    <parameter1>
+      <!-- Use variable interpolation to pass context to minimize generation of tokens, example: <content>$var1$<</content> -->
+      value1
+    </parameter1>
     <parameter2>value2</parameter2>
 </tool_name>
 ```
 
-### Tool Usage Guidelines
-1. Before Repeating a Tool Call:
-   - Review previous results in detail.
-   - State why a repeat is needed.
-   - Adjust parameters if necessary.
-   - Consider whether other tools are more appropriate.
-   - Use variable interpolation to pass context to minimize generation of tokens, example: <toolname>$var1$<</toolname>
-
-2. When Tool Calls Fail:
-   - Examine the error message carefully.
-   - Adjust parameters if needed.
-   - Consider alternative tools.
-   - Break down complex processes into smaller steps if necessary.
-
 ### Available Tools
 {tools}
 

quantalogic/tools/__init__.py

@@ -8,6 +8,7 @@ from .execute_bash_command_tool import ExecuteBashCommandTool
 from .input_question_tool import InputQuestionTool
 from .list_directory_tool import ListDirectoryTool
 from .llm_tool import LLMTool
+from .llm_vision_tool import LLMVisionTool
 from .markitdown_tool import MarkitdownTool
 from .nodejs_tool import NodeJsTool
 from .python_tool import PythonTool
@@ -30,6 +31,7 @@ __all__ = [
     "InputQuestionTool",
     "ListDirectoryTool",
     "LLMTool",
+    "LLMVisionTool",
     "ExecuteBashCommandTool",
     "PythonTool",
     "ElixirTool",

quantalogic/tools/llm_tool.py

@@ -17,11 +17,11 @@ class LLMTool(Tool):
     description: str = Field(
         default=(
             "Generates answers to questions using a specified language model. "
-            "Note: This tool operates in isolation and does not have access to: "
+            "Note: This tool operates in total isolation and does not have access to: "
             " - Memory: All context must be explicitly provided in the prompt. "
             " - File system."
             " - Variables: Any required variables should be interpolated into the prompt (e.g., $var1$). "
-            " - Other tools: It cannot invoke or interact with other tools. "
+            " - No access to Tools, URL, file, or other external resources. "
             "Ensure all necessary information is included directly in your prompt."
         )
     )

quantalogic/tools/llm_vision_tool.py

@@ -0,0 +1,140 @@
+"""LLM Vision Tool for analyzing images using a language model."""
+
+import logging
+from typing import Optional
+
+from pydantic import ConfigDict, Field
+
+from quantalogic.generative_model import GenerativeModel, Message
+from quantalogic.tools.tool import Tool, ToolArgument
+
+#DEFAULT_MODEL_NAME = "ollama/llama3.2-vision"
+DEFAULT_MODEL_NAME = "openrouter/openai/gpt-4o-mini"
+
+
+class LLMVisionTool(Tool):
+    """Tool to analyze images using a specified language model."""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
+
+    name: str = Field(default="llm_vision_tool")
+    description: str = Field(
+        default=(
+            "Analyzes images and generates responses using a specified language model. "
+            "Supports multimodal input combining text and images."
+        )
+    )
+    arguments: list = Field(
+        default=[
+            ToolArgument(
+                name="system_prompt",
+                arg_type="string",
+                description="The system prompt to guide the model's behavior",
+                required=True,
+                example="You are an expert in image analysis and visual understanding.",
+            ),
+            ToolArgument(
+                name="prompt",
+                arg_type="string",
+                description="The question or instruction about the image",
+                required=True,
+                example="What is shown in this image?",
+            ),
+            ToolArgument(
+                name="image_url",
+                arg_type="string",
+                description="URL of the image to analyze",
+                required=True,
+                example="https://example.com/image.jpg",
+            ),
+            ToolArgument(
+                name="temperature",
+                arg_type="string",
+                description='Sampling temperature between "0.0" and "1.0"',
+                required=True,
+                default="0.7",
+                example="0.7",
+            ),
+        ]
+    )
+
+    model_name: str = Field(..., description="The name of the language model to use")
+    generative_model: Optional[GenerativeModel] = Field(default=None)
+
+    def model_post_init(self, __context):
+        """Initialize the generative model after model initialization."""
+        if self.generative_model is None:
+            self.generative_model = GenerativeModel(model=self.model_name)
+            logging.debug(f"Initialized LLMVisionTool with model: {self.model_name}")
+
+    def execute(
+        self,
+        system_prompt: str,
+        prompt: str,
+        image_url: str,
+        temperature: str = "0.7"
+    ) -> str:
+        """Execute the tool to analyze an image and generate a response.
+
+        Args:
+            system_prompt: The system prompt to guide the model
+            prompt: The question or instruction about the image
+            image_url: URL of the image to analyze
+            temperature: Sampling temperature
+
+        Returns:
+            The generated response
+
+        Raises:
+            ValueError: If temperature is invalid or image_url is malformed
+            Exception: If there's an error during response generation
+        """
+        try:
+            temp = float(temperature)
+            if not (0.0 <= temp <= 1.0):
+                raise ValueError("Temperature must be between 0 and 1.")
+        except ValueError as ve:
+            logging.error(f"Invalid temperature value: {temperature}")
+            raise ValueError(f"Invalid temperature value: {temperature}") from ve
+
+        if not image_url.startswith(("http://", "https://")):
+            raise ValueError("Image URL must start with http:// or https://")
+
+        # Prepare the messages history
+        messages_history = [
+            Message(role="system", content=system_prompt),
+        ]
+
+        if self.generative_model is None:
+            self.generative_model = GenerativeModel(model=self.model_name)
+
+        self.generative_model.temperature = temp
+
+        try:
+            response_stats = self.generative_model.generate_with_history(
+                messages_history=messages_history,
+                prompt=prompt,
+                image_url=image_url
+            )
+            response = response_stats.response.strip()
+            logging.info(f"Generated response: {response}")
+            return response
+        except Exception as e:
+            logging.error(f"Error generating response: {e}")
+            raise Exception(f"Error generating response: {e}") from e
+
+
+if __name__ == "__main__":
+    # Example usage
+    tool = LLMVisionTool(model_name=DEFAULT_MODEL_NAME)
+    system_prompt = "You are a vision expert."
+    question = "What is shown in this image? Describe it with details."
+    image_url = "https://fastly.picsum.photos/id/767/200/300.jpg?hmac=j5YA1cRw-jS6fK3Mx2ooPwl2_TS3RSyLmFmiM9TqLC4"
+    temperature = "0.7"
+    answer = tool.execute(
+        system_prompt=system_prompt,
+        prompt=question,
+        image_url=image_url,
+        temperature=temperature
+    )
+    print(answer)

quantalogic/version.py

@@ -1,4 +1,4 @@
 
-VERSION = "0.2.8"
+VERSION = "0.2.10"
 def get_version() -> str:
     return VERSION

{quantalogic-0.2.8.dist-info → quantalogic-0.2.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quantalogic
-Version: 0.2.8
+Version: 0.2.10
 Summary: QuantaLogic ReAct Agents
 Author: Raphaël MANSUY
 Author-email: raphael.mansuy@gmail.com
@@ -123,8 +123,10 @@ Usage: quantalogic [OPTIONS] COMMAND [ARGS]...
 
 Options:
   --version                       Show version information.
-  --model-name TEXT               Specify the model to use (litellm format,
+  --model-name TEXT               Specify the text model to use (litellm format,
                                   e.g. "openrouter/deepseek-chat").
+  --vision-model-name TEXT        Specify the vision model to use (litellm format,
+                                  e.g. "openrouter/A/gpt-4o-mini").
   --log [info|debug|warning]      Set logging level (info/debug/warning).
   --verbose                       Enable verbose output.
   --mode [code|basic|interpreter|full|code-basic]
@@ -385,7 +387,7 @@ By integrating these tools into its architecture, QuantaLogic allows agents to p
 | Script Execution       | Python Tool, Node.js Tool, Elixir Tool                                                            |
 | File Operations        | Read File Tool, Write File Tool, Edit Whole Content Tool, Replace In File Tool                   |
 | Code Analysis          | Search Definition Names Tool, Ripgrep Tool                                                        |
-| Content Generation      | LLM Tool                                                                                          |
+| Content Generation      | LLM Tool, LLMVisionTool                                                                           |
 | Utility and Management  | Download HTTP File Tool, List Directory Tool, Markitdown Tool, Unified Diff Tool                 |
 
 ---
@@ -681,7 +683,30 @@ print("Ripgrep Results:", output)
 
 ---
 
-### 14. LLM Tool
+#### 14. LLMVisionTool
+
+The **LLMVisionTool** enables processing of visual inputs using vision-language models.
+
+##### Parameters
+
+| Parameter      | Type    | Description                                                            | Example                                    |
+|----------------|---------|------------------------------------------------------------------------|--------------------------------------------|
+| `image_path`   | string  | Path to the image file to process                                      | `./path/to/image.png`                      |
+| `prompt`       | string  | The question or instruction for the vision model                       | `Describe the contents of this image`      |
+| `temperature`  | float   | Sampling temperature between 0.0 and 1.0                              | `0.7`                                      |
+
+##### Example Usage
+```python
+vision_tool = LLMVisionTool()
+response = vision_tool.execute(
+    image_path="./path/to/image.png",
+    prompt="Describe the contents of this image",
+    temperature=0.7
+)
+print("Vision Model Response:", response)
+```
+
+#### 15. LLM Tool
 
 The **LLM Tool** generates answers using a specified language model.
 
@@ -706,7 +731,7 @@ print("LLM Response:", response)
 
 ---
 
-### 15. Download HTTP File Tool
+### 16. Download HTTP File Tool
 
 The **Download HTTP File Tool** downloads a file from a specified HTTP URL.
 
@@ -726,7 +751,7 @@ print(result)
 
 ---
 
-### 16. List Directory Tool
+### 17. List Directory Tool
 
 The **List Directory Tool** lists files in a specified directory.
 
@@ -745,7 +770,7 @@ print("Directory Files:", result)
 
 ---
 
-### 17. Markitdown Tool
+### 18. Markitdown Tool
 
 The **Markitdown Tool** processes markdown files, possibly for conversion or rendering.
 
@@ -762,26 +787,6 @@ result = markitdown_tool.execute(markdown_path="./path/to/file.md")
 print("Processed Markdown Output:", result)
 ```
 
----
-
-### 18. Unified Diff Tool
-
-The **Unified Diff Tool** generates a unified diff between two texts or files.
-
-#### Parameters
-
-| Parameter    | Type   | Description                                    | Example                |
-|--------------|--------|------------------------------------------------|------------------------|
-| `original`   | string | The original content or file path.            | `old_text.txt`         |
-| `updated`    | string | The updated content or file path.             | `new_text.txt`         |
-
-#### Example Usage
-```python
-diff_tool = UnifiedDiffTool()
-result = diff_tool.execute(original="old_text.txt", updated="new_text.txt")
-print("Unified Diff Output:", result)
-```
-
 
 #### Creating Custom Tools
 

{quantalogic-0.2.8.dist-info → quantalogic-0.2.10.dist-info}/RECORD

@@ -1,15 +1,15 @@
 quantalogic/__init__.py,sha256=HFk7_19UzHzYwvPzb9QTQ4w_lPwTTPda61AYb8qggZY,686
-quantalogic/agent.py,sha256=QBQc-C_h_DsstV2ntHETh8XywPAcY31-ThO2SiBqvB0,25443
-quantalogic/agent_config.py,sha256=sNuZ0Y02h3ra2hcjAE3-8Tt-NE8QSarRsTB0auTLgn0,3595
-quantalogic/coding_agent.py,sha256=ivHBn3hIDEt9V1LDXI4l-COeGXYVp_v_rrx9ng8sP2A,3214
+quantalogic/agent.py,sha256=s6kmgH2XLhtg1ZBl1SckV7ayFXkR_acs305Z2l1O2Bg,26680
+quantalogic/agent_config.py,sha256=Gm-i84dQF7CC1NUEtXpdw-qgZh5Jp1q4ni_SxMIxm-U,4518
+quantalogic/coding_agent.py,sha256=qT24jneNLUH1zMQUWBBTYDIVjCLPNm1iCaXB0v3w9yc,3469
 quantalogic/event_emitter.py,sha256=jqot2g4JRXc88K6PW837Oqxbf7shZfO-xdPaUWmzupk,7901
-quantalogic/generative_model.py,sha256=JkZz7YBd-v9n5EURidKUddkCLNvNZaqMzo5PgE_ugmc,8398
+quantalogic/generative_model.py,sha256=N5FjE0kMpKKdpP9_QEOZJ0rkOpYPaJWG91jzEyyovXA,10725
 quantalogic/interactive_text_editor.py,sha256=kYeTA2qej5kxtPvAUHy_Dr2MhrGQAyenLFpW9mU9Rmw,6855
-quantalogic/main.py,sha256=_oLhXNTDSH7Zfc1__jPgOvqZGpYH5VSjamdLwZ4xqDM,8698
+quantalogic/main.py,sha256=0zsr8-BnDPaR3roagL-3RX71hf4N0PqcgnCnJDO-gII,9626
 quantalogic/memory.py,sha256=zbtRuM05jaS2lJll-92dt5JfYVLERnF_m_9xqp2x-k0,6304
 quantalogic/model_names.py,sha256=UZlz25zG9B2dpfwdw_e1Gw5qFsKQ7iME9FJh9Ts4u6s,938
 quantalogic/print_event.py,sha256=-4qZmFI2BTkXuGE9DoKm6Vs-GzK1F9WJGt9GqpRQlQQ,2175
-quantalogic/prompts.py,sha256=GWVWnwFBxcVtOPvm1rUr1VnWY0OVqaBwVU_pMNunH2k,3609
+quantalogic/prompts.py,sha256=BHIST57DYcTeTb7rvV1QkGLt0_B8Wk8a_9tsnsN6suk,3547
 quantalogic/server/__init__.py,sha256=8sz_PYAUCrkM6JM5EAUeIzNM4NPW6j6UT72JVkc21WQ,91
 quantalogic/server/agent_server.py,sha256=GmglYf-LeVQQOdikMFDvPq1R0wKt6wIBpufW8XzP-iE,22489
 quantalogic/server/models.py,sha256=nVUGWElOsUw8QnRCGJylk25wCew_5gohe6nldYighUA,1322
@@ -19,7 +19,7 @@ quantalogic/server/static/js/event_visualizer.js,sha256=eFkkWyNZw3zOZlF18kxbfsWq
 quantalogic/server/static/js/quantalogic.js,sha256=x7TrlZGR1Y0WLK2DWl1xY847BhEWMPnL0Ua7KtOldUc,22311
 quantalogic/server/templates/index.html,sha256=nDnXJoQEm1vXbhXtgaYk0G5VXj0wwzE6KrqEDhHFpj4,7773
 quantalogic/tool_manager.py,sha256=FyghX2M_yGmdL7ovJR4ZGYIiBwkxA-bPjpI-y4IFx4Y,2421
-quantalogic/tools/__init__.py,sha256=VmvOHjCZmcVRnm5meMVBE8wMRmY1D7M2kxkrPeieWcI,1431
+quantalogic/tools/__init__.py,sha256=OxZp1nWZC5EewMJJVEvP6Fd2RMFlpaIYLHqChXPG_6s,1495
 quantalogic/tools/agent_tool.py,sha256=qeRp74EBqPSGu6JNZMATGyDoSCzPo7EnB2rmCP5wsBE,3050
 quantalogic/tools/download_http_file_tool.py,sha256=wTfanbXjIRi5-qrbluuLvNmDNhvmYAnlMVb3dO8C2ss,2210
 quantalogic/tools/edit_whole_content_tool.py,sha256=nXmpAvojvqvAcqNMy1kUKZ1ocboky_ZcnCR4SNCSPgw,2360
@@ -37,7 +37,8 @@ quantalogic/tools/language_handlers/rust_handler.py,sha256=t_AqKVa3KVk6SVkq_UjUU
 quantalogic/tools/language_handlers/scala_handler.py,sha256=wr-cWOIFOc0UYwODmEtT6rV63Qf1NyNB_BLo23GLrvk,1281
 quantalogic/tools/language_handlers/typescript_handler.py,sha256=L4vuJMYxKO3_83dQhdwZ9fogauIV7rwoicRT0xLGfkQ,1738
 quantalogic/tools/list_directory_tool.py,sha256=8Hy38DelSh-mRqS_uDLpeBYoHLtEy5ji77xI-TJu3Ms,4176
-quantalogic/tools/llm_tool.py,sha256=_6cMM2gpX6Wc_rV5y8L6-w3PoRwZtoRl09ZlPAlOckw,5005
+quantalogic/tools/llm_tool.py,sha256=ktwsg1LI3rSPS5XCwp-GAFiYCaXPSXRzUaqjOexPfEg,5011
+quantalogic/tools/llm_vision_tool.py,sha256=S99pKpIi8WUPREXXPArsuA7iTdXzUtiXg32J9NKCvlc,5028
 quantalogic/tools/markitdown_tool.py,sha256=GHJMPdWmwF-CBu3vHWhy-kXJYRDluFkh18KN06yNHOc,4101
 quantalogic/tools/nodejs_tool.py,sha256=2VTkZgtyXmv2E18CVaml3CKZE28WL9Tbv2IVdziv8wA,19903
 quantalogic/tools/python_tool.py,sha256=t66ge3xXS55-wJkddnVU9210TuDVoRI0Y-rsZwWxYIk,18154
@@ -59,11 +60,11 @@ quantalogic/utils/get_quantalogic_rules_content.py,sha256=fnEFTyClXzpI0MLaM-gB9R
 quantalogic/utils/git_ls.py,sha256=_aXg2TwqYv9CoOrhQ1gqHCqu1j8wOVigQNWbGncSDlM,4361
 quantalogic/utils/read_file.py,sha256=tSRVHk8dIP4nNLL89v5kRki4hOTjVyjbmuEb2zwvwCY,2077
 quantalogic/utils/read_http_text_content.py,sha256=1nRLQ9DHP_fKrm0rIEJBF0ROmB78e4lct2hUzD2PAUk,4408
-quantalogic/version.py,sha256=En2YsrG-VTf0ISc2bW9hm7HFnvQTMuxJpfpAaapjJDI,64
+quantalogic/version.py,sha256=rbj6GCYHf5rUvES_yzIMSiI_7I2h-zkrmZOjre8NI7A,65
 quantalogic/xml_parser.py,sha256=cTRorr5sVfkIzH72M0C-GQ9ROGPiz2FTT66U9ndjzhE,9538
 quantalogic/xml_tool_parser.py,sha256=lsVzClZBrZan7wjCuCKnGHWzksXI3VMy_vWthxu2_bo,3738
-quantalogic-0.2.8.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-quantalogic-0.2.8.dist-info/METADATA,sha256=XISo9oYxS-YXKmE7Prszx8Y_b4Oc36FYNXKAzhILsE8,39180
-quantalogic-0.2.8.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-quantalogic-0.2.8.dist-info/entry_points.txt,sha256=wgSq5SRU98yvlRHGEZD1Xn7sS5CSjH2RfUtTa6Qy28Q,52
-quantalogic-0.2.8.dist-info/RECORD,,
+quantalogic-0.2.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+quantalogic-0.2.10.dist-info/METADATA,sha256=Hu3vVRYMVGIPJpoCxP_6S4XKbw6R5rKPxcvS5y4Xfnw,39747
+quantalogic-0.2.10.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+quantalogic-0.2.10.dist-info/entry_points.txt,sha256=wgSq5SRU98yvlRHGEZD1Xn7sS5CSjH2RfUtTa6Qy28Q,52
+quantalogic-0.2.10.dist-info/RECORD,,