npcsh 0.3.31__py3-none-any.whl → 1.0.0__py3-none-any.whl

npcsh/plonk.py

@@ -1,291 +1,409 @@
-import json
+from npcpy.data.image import capture_screenshot
 import time
+import platform
+from npcpy.llm_funcs import get_llm_response
+from npcpy.work.desktop import perform_action, action_space
+from PIL import Image, ImageDraw, ImageFont
 
-try:
-    import pyautogui
-except KeyError as e:
-    print(f"Could not load pyautogui due to the following error: {e}")
-
-from .image import capture_screenshot
-from .llm_funcs import get_llm_response
-
-import subprocess
-import os
-
-
-from typing import Any
-
-action_space = {
-    "hotkey": {"key": "string"},  # For pressing hotkeys
-    "click": {
-        "x": "int between 0 and 100",
-        "y": "int between 0 and 100",
-    },  # For clicking
-    "drag": {
-        "x": "int between 0 and 100",
-        "y": "int between 0 and 100",
-        "duration": "int",
-    },  # For dragging
-    "wait": {"duration": "int"},  # For waiting
-    "type": {"text": "string"},
-    "right_click": {"x": "int between 0 and 100", "y": "int between 0 and 100"},
-    "double_click": {"x": "int between 0 and 100", "y": "int between 0 and 100"},
-    "bash": {"command": "string"},
-}
-
-
-def perform_action(action):
-    """
-    Execute different types of actions using PyAutoGUI
-    """
-    try:
-        pyautogui.PAUSE = 1  # Add a small pause between actions
-        pyautogui.FAILSAFE = (
-            True  # Enable fail-safe to stop script by moving mouse to corner
-        )
-
-        print(f"Action received: {action}")  # Debug print
-
-        if action["type"] == "click":
-            pyautogui.click(x=action.get("x"), y=action.get("y"))
-
-        elif action["type"] == "double_click":
-            pyautogui.doubleClick(x=action.get("x"), y=action.get("y"))
-
-        elif action["type"] == "right_click":
-            pyautogui.rightClick(x=action.get("x"), y=action.get("y"))
-
-        elif action["type"] == "drag":
-            pyautogui.dragTo(
-                x=action.get("x"), y=action.get("y"), duration=action.get("duration", 1)
-            )
-
-        elif action["type"] == "type":
-            text = action.get("text", "")
-            if isinstance(text, dict):
-                text = text.get("text", "")
-            pyautogui.typewrite(text)
-
-        elif action["type"] == "hotkey":
-            keys = action.get("text", "")
-            print(f"Hotkey action: {keys}")  # Debug print
-            if isinstance(keys, str):
-                keys = [keys]
-            elif isinstance(keys, dict):
-                keys = [keys.get("key", "")]
-            pyautogui.hotkey(*keys)
-
-        elif action["type"] == "wait":
-            time.sleep(action.get("duration", 1))  # Wait for the given time in seconds
-
-        elif action["type"] == "bash":
-            command = action.get("command", "")
-            print(f"Running bash command: {command}")  # Debug print
-            subprocess.Popen(
-                command, shell=True
-            )  # Run the command without waiting for it to complete
-            print(f"Bash Command Output: {result.stdout.decode()}")  # Debug output
-            print(f"Bash Command Error: {result.stderr.decode()}")  # Debug error
-
-        return {"status": "success"}
-
-    except Exception as e:
-        return {"status": "error", "message": str(e)}
-
-
-def plonk(request, action_space, model=None, provider=None, npc=None):
-    """
-    Main interaction loop with LLM for action determination
-
-    Args:
-        request (str): The task to be performed
-        action_space (dict): Available action types and the inputs they require
-        npc (optional): NPC object for context and screenshot
-        **kwargs: Additional arguments for LLM response
+def get_system_examples():
+    system = platform.system()
+    if system == "Windows":
+        return "Examples: start firefox, notepad, calc, explorer"
+    elif system == "Darwin":
+        return "Examples: open -a Firefox, open -a TextEdit, open -a Calculator"
+    else:
+        return "Examples: firefox &, gedit &, gnome-calculator &"
+
+def execute_plonk_command(request, action_space, model, provider, npc=None, max_iterations=10, debug=False):
+    synthesized_summary = []
+
+    """Synthesizes information gathered during the computer use run and logs key data points for
+    analysis. This function can be extended to store or report the synthesized knowledge as required.
     """
-    prompt = f"""
-    Here is a request from a user:
-    {request}
-
-    Your job is to choose certain actions, take screenshots,
-    and evaluate what the next step is to complete the task.
-
-    You can choose from the following action types:
-    {json.dumps(action_space)}
-
-
-
-    Attached to the message is a screenshot of the current screen.
-
-    Please use that information to determine the next steps.
-    Your response must be a JSON with an 'actions' key containing a list of actions.
-    Each action should have a 'type' and any necessary parameters.https://www.reddit.com
-
-
-    For example:
-        Your response should look like:
 
+    system = platform.system()
+    system_examples = get_system_examples()
+    
+    messages = []
+    last_action_feedback = "None"
+    last_click_coords = None
+
+    iteration_count = 0
+    while iteration_count < max_iterations:
+        # Gathering summary of actions performed this iteration
+        synthesized_info = {
+            'iteration': iteration_count + 1,
+            'last_action_feedback': last_action_feedback,
+            'last_click_coords': last_click_coords
+        }
+        synthesized_summary.append(synthesized_info)
+
+        if debug:
+            print(f"Synthesized info at iteration {iteration_count + 1}: {synthesized_info}")
+
+        if debug:
+            print(f"Iteration {iteration_count + 1}/{max_iterations}")
+
+        # YOUR PROMPT, UNTOUCHED
+        prompt_template = f"""
+        Goal: {request}
+        Feedback from last action: {last_action_feedback}
+
+        Your task is to control the computer to achieve the goal.
+        
+        THOUGHT PROCESS:
+        1. Analyze the screen. Is the application I need (e.g., a web browser) already open?
+        2. If YES, `click` it. If NO, use `bash` to launch it. Use the examples: {system_examples}.
+        
+        
+        CRITICAL COMPLETION RULE:
+        Once the goal is visually complete on the screen, your ONLY next action is to use the 'quit' action.
+
+        Your response MUST be a JSON object with an "actions" key.
+        All clicking actions should use percentage coordinates relative 
+        to the screen size, as we will
+        manually translate them to the proper screen size. 
+        your x and y values for clicks must ALWAYS be between 0 and 100.
+        The x and y are (0,0) at the TOP LEFT CORNER OF THE SCREEN.
+        The bottom right corner of the screen is (100,100).
+        the bottom left corner is (0,100) and the top right corner is (100,0).
+        
+        
+        
+        
+        ---
+        EXAMPLE 1: Task "Create and save a file named 'memo.txt' with the text 'Meeting at 3pm'"
         {{
-            "actions": [
-                {{"type":"bash", "command":"firefox &"}},
-                {{"type": "click", "x": 5, "y": 5}},
-                {{'type': 'type', 'text': 'https://www.google.com'}}
-                ]
-                }}
-
-    IF you have to type something, ensure that it iis first opened and selected. Do not
-    begin with typing immediately.
-    If you have to click, the numbers should range from 0 to 100 in x and y  with 0,0 being in the upper left.
-
-
-    IF you have accomplished the task, return an empty list.
-    Do not include any additional markdown formatting.
-    """
-
-    while True:
-        # Capture screenshot using NPC-based method
-        screenshot = capture_screenshot(npc=npc, full=True)
-
-        # Ensure screenshot was captured successfully
-        if not screenshot:
-            print("Screenshot capture failed")
-            return None
-
-        # Get LLM response
+          "actions": [
+            {{ "type": "bash", "command": "gedit &" }},
+            {{ "type": "wait", "duration": 2 }},
+            {{'type':'click', 'x': 10, 'y': 30}},  
+            {{ "type": "type", "text": "Meeting at 3pm" }},
+            {{ "type": "hotkey", "keys": ["ctrl", "s"] }},
+            {{ "type": "wait", "duration": 1 }},
+            {{ "type": "type", "text": "memo.txt" }},
+            {{ "type": "key", "keys": ["enter"] }},
+          ]
+        }}
+        ---
+        EXAMPLE 2: Task "Search for news about space exploration"
+        {{
+          "actions": [
+            {{ "type": "bash", "command": "firefox &" }},
+            {{ "type": "wait", "duration": 3 }},
+            {{ "type": "type", "text": "news about space exploration" }},
+            {{ "type": "key", "keys": ["enter"] }},
+          ]
+        }}
+        
+        ---
+        
+        Once a task has been verified and completed, your action list should only be 
+        {{
+          "actions": [
+            {{ "type": "quit" }}
+          ]
+        }}
+        """
+
+        screenshot_path = capture_screenshot(npc=npc, full=True).get('file_path')
+        if not screenshot_path:
+            time.sleep(2)
+            continue
+
+        image_to_send_path = screenshot_path
+        if last_click_coords:
+            try:
+                img = Image.open(screenshot_path)
+                draw = ImageDraw.Draw(img)
+                width, height = img.size
+                x_pixel = int(last_click_coords['x'] * width / 100)
+                y_pixel = int(last_click_coords['y'] * height / 100)
+                
+                try:
+                    font = ImageFont.truetype("DejaVuSans-Bold.ttf", size=48)
+                except IOError:
+                    font = ImageFont.load_default()
+
+                draw.text((x_pixel - 8, y_pixel - 12),
+                          f"+{last_click_coords['x'],last_click_coords['y']}",
+                          fill="red",
+                          font=font)
+                
+                marked_image_path = "/tmp/marked_screenshot.png"
+                img.save(marked_image_path)
+                image_to_send_path = marked_image_path
+                print(f"Drew marker at ({x_pixel}, {y_pixel}) on new screenshot.")
+            except Exception as e:
+                print(f"Failed to draw marker on image: {e}")
+        
         response = get_llm_response(
-            prompt,
-            images=[screenshot],
+            prompt=prompt_template,
             model=model,
             provider=provider,
             npc=npc,
+            images=[image_to_send_path],
+            messages=messages,
             format="json",
         )
-        # print("LLM Response:", response, type(response))
-        # Check if task is complete
-        print(response["response"])
-        if not response["response"].get("actions", []):
-            return response
-
-        # Execute actions
-        for action in response["response"]["actions"]:
-            print("Performing action:", action)
-            action_result = perform_action(action)
-            perform_action({"type": "wait", "duration": 5})
 
-            # Optional: Add error handling or logging
-            if action_result.get("status") == "error":
-                print(f"Error performing action: {action_result.get('message')}")
+        if "messages" in response:
+            messages = response["messages"]
+        
+        response_data = response.get('response')
+
+        if not isinstance(response_data, dict) or "actions" not in response_data:
+            last_action_feedback = f"Invalid JSON response from model: {response_data}"
+            continue
+
+        actions_list = response_data.get("actions", [])
+        
+        if not isinstance(actions_list, list):
+            last_action_feedback = "Model did not return a list in the 'actions' key."
+            continue
+        
+        # Reset last click before processing new actions
+        last_click_coords = None
+        for action in actions_list:
+            if debug:
+                print(f"Executing action: {action}")
+            if action.get("type") == "quit":
+                print("Task complete: Model returned 'quit' action.")
+                return "SUCCESS"
+                
+            result = perform_action(action)
+            last_action_feedback = result.get("message") or result.get("output")
+
+            if action.get("type") == "click":
+                last_click_coords = {"x": action.get("x"), "y": action.get("y")}
+            
+            if result.get("status") == "error":
+                print(f"Action failed, providing feedback to model: {last_action_feedback}")
+                break 
+            time.sleep(1)
+        
+        if not actions_list:
+            last_action_feedback = "No actions were returned. The task is likely not complete. Re-evaluating."
+            print(last_action_feedback)
+        
+        iteration_count += 1
+    
+    return None
+
+def synthesize_and_display_summary(synthesized_summary, debug=False):
+    """Synthesizes information gathered during the computer use run and logs key data points."""
+    if not synthesized_summary:
+        print("No synthesized info to display.")
+        return
+
+    print("\nSynthesized Summary of Computer Use Run:")
+    for info in synthesized_summary:
+        print(f"Iteration {info['iteration']}:\n"
+              f"  Last Action Feedback: {info['last_action_feedback']}\n"
+              f"  Last Click Coordinates: {info['last_click_coords']}")
+    print("End of synthesized summary.\n")
+
+
+
+def repl_loop():
+    print("Assistant REPL - Type your plonk command or 'exit' to quit.")
+    while True:
+        user_input = input("Enter your command: ").strip()
+        if user_input.lower() == 'exit':
+            print("Exiting REPL. Goodbye!")
+            break
+        if not user_input:
+            continue
+
+        # Run the plonk command and get synthesized summary
+        synthesized_summary = execute_plonk_command(
+            request=user_input,
+            action_space=action_space,
+            model="gpt-4o-mini",
+            provider="openai",
+            max_iterations=8,
+            debug=True
+        )
 
-        # Small delay between action batches
-        time.sleep(1)
+        if synthesized_summary and isinstance(synthesized_summary, list):
+            print("Command executed with synthesized summary.")
+            synthesize_and_display_summary(synthesized_summary)
+        else:
+            print("Command did not complete within iteration limit or returned no summary.")
 
 
-def test_open_reddit(npc: Any = None):
+def execute_plonk_command(request, action_space, model, provider, npc=None, max_iterations=10, debug=False):
+    """Synthesizes information gathered during the computer use run and logs key data points for
+    analysis. This function can be extended to store or report the synthesized knowledge as required.
     """
-    Test function to open a web browser and navigate to Reddit using plonk
-    """
-    # Define the action space for web navigation
-
-    # Request to navigate to Reddit
-    request = "Open a web browser and go to reddit.com"
-
-    # Determine the browser launch hotkey based on the operating system
-    import platform
 
     system = platform.system()
+    system_examples = get_system_examples()
+    
+    messages = []
+    last_action_feedback = "None"
+    last_click_coords = None
+
+    iteration_count = 0
+
+    synthesized_summary = []
+
+    while iteration_count < max_iterations:
+        synthesized_info = {
+            'iteration': iteration_count + 1,
+            'last_action_feedback': last_action_feedback,
+            'last_click_coords': last_click_coords
+        }
+        synthesized_summary.append(synthesized_info)
+
+        if debug:
+            print(f"Synthesized info at iteration {iteration_count + 1}: {synthesized_info}")
+
+        if debug:
+            print(f"Iteration {iteration_count + 1}/{max_iterations}")
+
+        prompt_template = f"""
+        Goal: {request}
+        Feedback from last action: {last_action_feedback}
+
+        Your task is to control the computer to achieve the goal.
+        
+        THOUGHT PROCESS:
+        1. Analyze the screen. Is the application I need (e.g., a web browser) already open?
+        2. If YES, `click` it. If NO, use `bash` to launch it. Use the examples: {system_examples}.
+        
+        
+        CRITICAL COMPLETION RULE:
+        Once the goal is visually complete on the screen, your ONLY next action is to use the 'quit' action.
+
+        Your response MUST be a JSON object with an "actions" key.
+        All clicking actions should use percentage coordinates relative 
+        to the screen size, as we will
+        manually translate them to the proper screen size. 
+        your x and y values for clicks must ALWAYS be between 0 and 100.
+        The x and y are (0,0) at the TOP LEFT CORNER OF THE SCREEN.
+        The bottom right corner of the screen is (100,100).
+        the bottom left corner is (0,100) and the top right corner is (100,0).
+        
+        
+        
+        ---
+        EXAMPLE 1: Task "Create and save a file named 'memo.txt' with the text 'Meeting at 3pm'"
+        {{
+          "actions": [
+            {{ "type": "bash", "command": "gedit &" }},
+            {{ "type": "wait", "duration": 2 }},
+            {{'type':'click', 'x': 10, 'y': 30}},  
+            {{ "type": "type", "text": "Meeting at 3pm" }},
+            {{ "type": "hotkey", "keys": ["ctrl", "s"] }},
+            {{ "type": "wait", "duration": 1 }},
+            {{ "type": "type", "text": "memo.txt" }},
+            {{ "type": "key", "keys": ["enter"] }},
+          ]
+        }}
+        ---
+        EXAMPLE 2: Task "Search for news about space exploration"
+        {{
+          "actions": [
+            {{ "type": "bash", "command": "firefox &" }},
+            {{ "type": "wait", "duration": 3 }},
+            {{ "type": "type", "text": "news about space exploration" }},
+            {{ "type": "key", "keys": ["enter"] }},
+          ]
+        }}
+        
+        ---
+        
+        Once a task has been verified and completed, your action list should only be 
+        {{
+          "actions": [
+            {{ "type": "quit" }}
+          ]
+        }}
+        """
+
+        screenshot_path = capture_screenshot(npc=npc, full=True).get('file_path')
+        if not screenshot_path:
+            time.sleep(2)
+            continue
+
+        image_to_send_path = screenshot_path
+        if last_click_coords:
+            try:
+                img = Image.open(screenshot_path)
+                draw = ImageDraw.Draw(img)
+                width, height = img.size
+                x_pixel = int(last_click_coords['x'] * width / 100)
+                y_pixel = int(last_click_coords['y'] * height / 100)
+                
+                try:
+                    font = ImageFont.truetype("DejaVuSans-Bold.ttf", size=48)
+                except IOError:
+                    font = ImageFont.load_default()
+
+                draw.text((x_pixel - 8, y_pixel - 12),
+                          f"+{last_click_coords['x'],last_click_coords['y']}",
+                          fill="red",
+                          font=font)
+                
+                marked_image_path = "/tmp/marked_screenshot.png"
+                img.save(marked_image_path)
+                image_to_send_path = marked_image_path
+                print(f"Drew marker at ({x_pixel}, {y_pixel}) on new screenshot.")
+            except Exception as e:
+                print(f"Failed to draw marker on image: {e}")
+        
+        response = get_llm_response(
+            prompt=prompt_template,
+            model=model,
+            provider=provider,
+            npc=npc,
+            images=[image_to_send_path],
+            messages=messages,
+            format="json",
+        )
 
-    if system == "Darwin":  # macOS
-        browser_launch_keys = ["command", "space"]
-        browser_search = "chrome"
-    elif system == "Windows":
-        browser_launch_keys = ["win", "r"]
-        browser_search = "chrome"
-    else:  # Linux or other
-        browser_launch_keys = ["alt", "f2"]
-        browser_search = "firefox"
-
-    # Perform the task using plonk
-    result = plonk(
-        request,
-        action_space,
-        model="gpt-4o-mini",
-        provider="openai",
-    )
-
-    # Optionally, you can add assertions or print results
-    print("Reddit navigation test result:", result)
-
-    return result
-
-
-def generate_plonk(
-    request,
-):
-    prompt = f"""
-
-    A user asked the following question: {request}
-
-    You are in charge of creating a plonk plan that will handle their request.
-    This plonk plan will be a series of steps that you will write that will be
-    used to generate a fully functioning system that will accomplish the user's request.
-    your plonk plan should be a python script that generates LLM prompts
-    that will be used to generate the distinct pieces of software.
-
-    The goal here is modularization, abstraction, separation of scales.
-    A careful set of instructions can pave the way for a system that can be iterated on
-    and improved with successive steps.
-
-    Here is an example of a question and answer that you might generate:
-
-    Question: "Set up an automation system that will open a web browser every morning
-        and go to my bank account and export my transactions."
-
-    Answer:
-    "{{'plonk plan': ```
-
-from npcsh.llm_funcs import get_llm_response
-
-automation_script = get_llm_response( '''
-    Write a python script that will request input from a user about what bank they use. Then use selenium to open the browser and navigate to the bank's website.
-    Get the user's username and password and log in, also through raw input.
-    Then navigate to the transactions page and export the transactions. Ensure you are sufficiently logging information at each step of the way so that the results can be
-    debugged efficiently.
-    Return the script without any additional comment or Markdown formatting. It is imperative that you do not include any additional text.
-''')
-# write the automation script to a file
-automation_script_file = open('automation_script.py', 'w')
-automation_script_file.write(automation_script)
-automation_script_file.close()
-
-
-scheduling_script = get_llm_response( f'''
-    Write a bash script that will set up an OS scheduler to run the automation script every morning at 8 am.
-    The automation script is located at ./automation_script.py.
-    You'll need to ensure that the full path is used in the scheduling script.
-    Return the script without any additional comment or Markdown formatting.
-    It is imperative that you do not include any additional text.
-    Do not leave any placeholder paths or variables in the script.
-    They must be able to execute without
-    any further modification by you or the user.
-    ''')
-# write the scheduling script to a file
-scheduling_script_file = open('scheduling_script.sh', 'w')
-scheduling_script_file.write(scheduling_script)
-
-scheduling_script_file.close()
-# attempt to run the scheduling script
-import subprocess
-subprocess.run(['bash', 'scheduling_script.sh'])
-```}}
-
-    In this example, we have set up a plan that will require multiple other LLM calls to generate the necessary items to
-    accomplish the user's request.
-
-    """
-
-    return get_llm_response(prompt)
+        if "messages" in response:
+            messages = response["messages"]
+        
+        response_data = response.get('response')
+
+        if not isinstance(response_data, dict) or "actions" not in response_data:
+            last_action_feedback = f"Invalid JSON response from model: {response_data}"
+            continue
+
+        actions_list = response_data.get("actions", [])
+        
+        if not isinstance(actions_list, list):
+            last_action_feedback = "Model did not return a list in the 'actions' key."
+            continue
+        
+        last_click_coords = None
+        for action in actions_list:
+            if debug:
+                print(f"Executing action: {action}")
+            if action.get("type") == "quit":
+                print("Task complete: Model returned 'quit' action.")
+                return synthesized_summary
+                
+            result = perform_action(action)
+            last_action_feedback = result.get("message") or result.get("output")
+
+            if action.get("type") == "click":
+                last_click_coords = {"x": action.get("x"), "y": action.get("y")}
+            
+            if result.get("status") == "error":
+                print(f"Action failed, providing feedback to model: {last_action_feedback}")
+                break 
+            time.sleep(1)
+        
+        if not actions_list:
+            last_action_feedback = "No actions were returned. The task is likely not complete. Re-evaluating."
+            print(last_action_feedback)
+        
+        iteration_count += 1
+    return synthesized_summary
 
 
-# Optional: If you want to run this as a standalone script
 if __name__ == "__main__":
-    test_open_reddit()
+    repl_loop()