npcpy 1.0.26__py3-none-any.whl → 1.2.32__py3-none-any.whl

npcpy/modes/plonk.py

@@ -1,409 +0,0 @@
-from npcpy.data.image import capture_screenshot
-import time
-import platform
-from npcpy.llm_funcs import get_llm_response
-from npcpy.work.desktop import perform_action, action_space
-from PIL import Image, ImageDraw, ImageFont
-
-def get_system_examples():
-    system = platform.system()
-    if system == "Windows":
-        return "Examples: start firefox, notepad, calc, explorer"
-    elif system == "Darwin":
-        return "Examples: open -a Firefox, open -a TextEdit, open -a Calculator"
-    else:
-        return "Examples: firefox &, gedit &, gnome-calculator &"
-
-def execute_plonk_command(request, action_space, model, provider, npc=None, max_iterations=10, debug=False):
-    synthesized_summary = []
-
-    """Synthesizes information gathered during the computer use run and logs key data points for
-    analysis. This function can be extended to store or report the synthesized knowledge as required.
-    """
-
-    system = platform.system()
-    system_examples = get_system_examples()
-    
-    messages = []
-    last_action_feedback = "None"
-    last_click_coords = None
-
-    iteration_count = 0
-    while iteration_count < max_iterations:
-        # Gathering summary of actions performed this iteration
-        synthesized_info = {
-            'iteration': iteration_count + 1,
-            'last_action_feedback': last_action_feedback,
-            'last_click_coords': last_click_coords
-        }
-        synthesized_summary.append(synthesized_info)
-
-        if debug:
-            print(f"Synthesized info at iteration {iteration_count + 1}: {synthesized_info}")
-
-        if debug:
-            print(f"Iteration {iteration_count + 1}/{max_iterations}")
-
-        # YOUR PROMPT, UNTOUCHED
-        prompt_template = f"""
-        Goal: {request}
-        Feedback from last action: {last_action_feedback}
-
-        Your task is to control the computer to achieve the goal.
-        
-        THOUGHT PROCESS:
-        1. Analyze the screen. Is the application I need (e.g., a web browser) already open?
-        2. If YES, `click` it. If NO, use `bash` to launch it. Use the examples: {system_examples}.
-        
-        
-        CRITICAL COMPLETION RULE:
-        Once the goal is visually complete on the screen, your ONLY next action is to use the 'quit' action.
-
-        Your response MUST be a JSON object with an "actions" key.
-        All clicking actions should use percentage coordinates relative 
-        to the screen size, as we will
-        manually translate them to the proper screen size. 
-        your x and y values for clicks must ALWAYS be between 0 and 100.
-        The x and y are (0,0) at the TOP LEFT CORNER OF THE SCREEN.
-        The bottom right corner of the screen is (100,100).
-        the bottom left corner is (0,100) and the top right corner is (100,0).
-        
-        
-        
-        
-        ---
-        EXAMPLE 1: Task "Create and save a file named 'memo.txt' with the text 'Meeting at 3pm'"
-        {{
-          "actions": [
-            {{ "type": "bash", "command": "gedit &" }},
-            {{ "type": "wait", "duration": 2 }},
-            {{'type':'click', 'x': 10, 'y': 30}},  
-            {{ "type": "type", "text": "Meeting at 3pm" }},
-            {{ "type": "hotkey", "keys": ["ctrl", "s"] }},
-            {{ "type": "wait", "duration": 1 }},
-            {{ "type": "type", "text": "memo.txt" }},
-            {{ "type": "key", "keys": ["enter"] }},
-          ]
-        }}
-        ---
-        EXAMPLE 2: Task "Search for news about space exploration"
-        {{
-          "actions": [
-            {{ "type": "bash", "command": "firefox &" }},
-            {{ "type": "wait", "duration": 3 }},
-            {{ "type": "type", "text": "news about space exploration" }},
-            {{ "type": "key", "keys": ["enter"] }},
-          ]
-        }}
-        
-        ---
-        
-        Once a task has been verified and completed, your action list should only be 
-        {{
-          "actions": [
-            {{ "type": "quit" }}
-          ]
-        }}
-        """
-
-        screenshot_path = capture_screenshot(npc=npc, full=True).get('file_path')
-        if not screenshot_path:
-            time.sleep(2)
-            continue
-
-        image_to_send_path = screenshot_path
-        if last_click_coords:
-            try:
-                img = Image.open(screenshot_path)
-                draw = ImageDraw.Draw(img)
-                width, height = img.size
-                x_pixel = int(last_click_coords['x'] * width / 100)
-                y_pixel = int(last_click_coords['y'] * height / 100)
-                
-                try:
-                    font = ImageFont.truetype("DejaVuSans-Bold.ttf", size=48)
-                except IOError:
-                    font = ImageFont.load_default()
-
-                draw.text((x_pixel - 8, y_pixel - 12),
-                          f"+{last_click_coords['x'],last_click_coords['y']}",
-                          fill="red",
-                          font=font)
-                
-                marked_image_path = "/tmp/marked_screenshot.png"
-                img.save(marked_image_path)
-                image_to_send_path = marked_image_path
-                print(f"Drew marker at ({x_pixel}, {y_pixel}) on new screenshot.")
-            except Exception as e:
-                print(f"Failed to draw marker on image: {e}")
-        
-        response = get_llm_response(
-            prompt=prompt_template,
-            model=model,
-            provider=provider,
-            npc=npc,
-            images=[image_to_send_path],
-            messages=messages,
-            format="json",
-        )
-
-        if "messages" in response:
-            messages = response["messages"]
-        
-        response_data = response.get('response')
-
-        if not isinstance(response_data, dict) or "actions" not in response_data:
-            last_action_feedback = f"Invalid JSON response from model: {response_data}"
-            continue
-
-        actions_list = response_data.get("actions", [])
-        
-        if not isinstance(actions_list, list):
-            last_action_feedback = "Model did not return a list in the 'actions' key."
-            continue
-        
-        # Reset last click before processing new actions
-        last_click_coords = None
-        for action in actions_list:
-            if debug:
-                print(f"Executing action: {action}")
-            if action.get("type") == "quit":
-                print("Task complete: Model returned 'quit' action.")
-                return "SUCCESS"
-                
-            result = perform_action(action)
-            last_action_feedback = result.get("message") or result.get("output")
-
-            if action.get("type") == "click":
-                last_click_coords = {"x": action.get("x"), "y": action.get("y")}
-            
-            if result.get("status") == "error":
-                print(f"Action failed, providing feedback to model: {last_action_feedback}")
-                break 
-            time.sleep(1)
-        
-        if not actions_list:
-            last_action_feedback = "No actions were returned. The task is likely not complete. Re-evaluating."
-            print(last_action_feedback)
-        
-        iteration_count += 1
-    
-    return None
-
-def synthesize_and_display_summary(synthesized_summary, debug=False):
-    """Synthesizes information gathered during the computer use run and logs key data points."""
-    if not synthesized_summary:
-        print("No synthesized info to display.")
-        return
-
-    print("\nSynthesized Summary of Computer Use Run:")
-    for info in synthesized_summary:
-        print(f"Iteration {info['iteration']}:\n"
-              f"  Last Action Feedback: {info['last_action_feedback']}\n"
-              f"  Last Click Coordinates: {info['last_click_coords']}")
-    print("End of synthesized summary.\n")
-
-
-
-def repl_loop():
-    print("Assistant REPL - Type your plonk command or 'exit' to quit.")
-    while True:
-        user_input = input("Enter your command: ").strip()
-        if user_input.lower() == 'exit':
-            print("Exiting REPL. Goodbye!")
-            break
-        if not user_input:
-            continue
-
-        # Run the plonk command and get synthesized summary
-        synthesized_summary = execute_plonk_command(
-            request=user_input,
-            action_space=action_space,
-            model="gpt-4o-mini",
-            provider="openai",
-            max_iterations=8,
-            debug=True
-        )
-
-        if synthesized_summary and isinstance(synthesized_summary, list):
-            print("Command executed with synthesized summary.")
-            synthesize_and_display_summary(synthesized_summary)
-        else:
-            print("Command did not complete within iteration limit or returned no summary.")
-
-
-def execute_plonk_command(request, action_space, model, provider, npc=None, max_iterations=10, debug=False):
-    """Synthesizes information gathered during the computer use run and logs key data points for
-    analysis. This function can be extended to store or report the synthesized knowledge as required.
-    """
-
-    system = platform.system()
-    system_examples = get_system_examples()
-    
-    messages = []
-    last_action_feedback = "None"
-    last_click_coords = None
-
-    iteration_count = 0
-
-    synthesized_summary = []
-
-    while iteration_count < max_iterations:
-        synthesized_info = {
-            'iteration': iteration_count + 1,
-            'last_action_feedback': last_action_feedback,
-            'last_click_coords': last_click_coords
-        }
-        synthesized_summary.append(synthesized_info)
-
-        if debug:
-            print(f"Synthesized info at iteration {iteration_count + 1}: {synthesized_info}")
-
-        if debug:
-            print(f"Iteration {iteration_count + 1}/{max_iterations}")
-
-        prompt_template = f"""
-        Goal: {request}
-        Feedback from last action: {last_action_feedback}
-
-        Your task is to control the computer to achieve the goal.
-        
-        THOUGHT PROCESS:
-        1. Analyze the screen. Is the application I need (e.g., a web browser) already open?
-        2. If YES, `click` it. If NO, use `bash` to launch it. Use the examples: {system_examples}.
-        
-        
-        CRITICAL COMPLETION RULE:
-        Once the goal is visually complete on the screen, your ONLY next action is to use the 'quit' action.
-
-        Your response MUST be a JSON object with an "actions" key.
-        All clicking actions should use percentage coordinates relative 
-        to the screen size, as we will
-        manually translate them to the proper screen size. 
-        your x and y values for clicks must ALWAYS be between 0 and 100.
-        The x and y are (0,0) at the TOP LEFT CORNER OF THE SCREEN.
-        The bottom right corner of the screen is (100,100).
-        the bottom left corner is (0,100) and the top right corner is (100,0).
-        
-        
-        
-        ---
-        EXAMPLE 1: Task "Create and save a file named 'memo.txt' with the text 'Meeting at 3pm'"
-        {{
-          "actions": [
-            {{ "type": "bash", "command": "gedit &" }},
-            {{ "type": "wait", "duration": 2 }},
-            {{'type':'click', 'x': 10, 'y': 30}},  
-            {{ "type": "type", "text": "Meeting at 3pm" }},
-            {{ "type": "hotkey", "keys": ["ctrl", "s"] }},
-            {{ "type": "wait", "duration": 1 }},
-            {{ "type": "type", "text": "memo.txt" }},
-            {{ "type": "key", "keys": ["enter"] }},
-          ]
-        }}
-        ---
-        EXAMPLE 2: Task "Search for news about space exploration"
-        {{
-          "actions": [
-            {{ "type": "bash", "command": "firefox &" }},
-            {{ "type": "wait", "duration": 3 }},
-            {{ "type": "type", "text": "news about space exploration" }},
-            {{ "type": "key", "keys": ["enter"] }},
-          ]
-        }}
-        
-        ---
-        
-        Once a task has been verified and completed, your action list should only be 
-        {{
-          "actions": [
-            {{ "type": "quit" }}
-          ]
-        }}
-        """
-
-        screenshot_path = capture_screenshot(npc=npc, full=True).get('file_path')
-        if not screenshot_path:
-            time.sleep(2)
-            continue
-
-        image_to_send_path = screenshot_path
-        if last_click_coords:
-            try:
-                img = Image.open(screenshot_path)
-                draw = ImageDraw.Draw(img)
-                width, height = img.size
-                x_pixel = int(last_click_coords['x'] * width / 100)
-                y_pixel = int(last_click_coords['y'] * height / 100)
-                
-                try:
-                    font = ImageFont.truetype("DejaVuSans-Bold.ttf", size=48)
-                except IOError:
-                    font = ImageFont.load_default()
-
-                draw.text((x_pixel - 8, y_pixel - 12),
-                          f"+{last_click_coords['x'],last_click_coords['y']}",
-                          fill="red",
-                          font=font)
-                
-                marked_image_path = "/tmp/marked_screenshot.png"
-                img.save(marked_image_path)
-                image_to_send_path = marked_image_path
-                print(f"Drew marker at ({x_pixel}, {y_pixel}) on new screenshot.")
-            except Exception as e:
-                print(f"Failed to draw marker on image: {e}")
-        
-        response = get_llm_response(
-            prompt=prompt_template,
-            model=model,
-            provider=provider,
-            npc=npc,
-            images=[image_to_send_path],
-            messages=messages,
-            format="json",
-        )
-
-        if "messages" in response:
-            messages = response["messages"]
-        
-        response_data = response.get('response')
-
-        if not isinstance(response_data, dict) or "actions" not in response_data:
-            last_action_feedback = f"Invalid JSON response from model: {response_data}"
-            continue
-
-        actions_list = response_data.get("actions", [])
-        
-        if not isinstance(actions_list, list):
-            last_action_feedback = "Model did not return a list in the 'actions' key."
-            continue
-        
-        last_click_coords = None
-        for action in actions_list:
-            if debug:
-                print(f"Executing action: {action}")
-            if action.get("type") == "quit":
-                print("Task complete: Model returned 'quit' action.")
-                return synthesized_summary
-                
-            result = perform_action(action)
-            last_action_feedback = result.get("message") or result.get("output")
-
-            if action.get("type") == "click":
-                last_click_coords = {"x": action.get("x"), "y": action.get("y")}
-            
-            if result.get("status") == "error":
-                print(f"Action failed, providing feedback to model: {last_action_feedback}")
-                break 
-            time.sleep(1)
-        
-        if not actions_list:
-            last_action_feedback = "No actions were returned. The task is likely not complete. Re-evaluating."
-            print(last_action_feedback)
-        
-        iteration_count += 1
-    return synthesized_summary
-
-
-if __name__ == "__main__":
-    repl_loop()

npcpy/modes/pti.py

@@ -1,234 +0,0 @@
-
-# pti
-import json 
-from typing import Dict, List, Optional, Any, Generator 
-import os 
-from npcpy.memory.command_history import CommandHistory, save_attachment_to_message, start_new_conversation,save_conversation_message
-from npcpy.npc_sysenv import (NPCSH_REASONING_MODEL,
-                              NPCSH_REASONING_PROVIDER, 
-                              NPCSH_CHAT_MODEL, 
-                              NPCSH_CHAT_PROVIDER, 
-                              NPCSH_API_URL, 
-                              NPCSH_STREAM_OUTPUT,print_and_process_stream_with_markdown)
-from npcpy.llm_funcs import get_llm_response, handle_request_input
-
-from npcpy.npc_compiler import NPC 
-from npcpy.data.load import load_csv, load_pdf
-from npcpy.data.text import rag_search
-
-
-
-
-
-
-def enter_reasoning_human_in_the_loop(
-    user_input=None,
-    messages: List[Dict[str, str]] = None,
-    reasoning_model: str = NPCSH_REASONING_MODEL,
-    reasoning_provider: str = NPCSH_REASONING_PROVIDER,
-    files : List = None, 
-    npc: Any = None,
-    conversation_id : str= False,
-    answer_only: bool = False,
-    context=None,
-) :
-    """
-    Stream responses while checking for think tokens and handling human input when needed.
-
-    Args:
-        messages: List of conversation messages
-        model: LLM model to use
-        provider: Model provider
-        npc: NPC instance if applicable
-
-    """
-    # Get the initial stream
-    loaded_content = {}  # New dictionary to hold loaded content
-
-    # Create conversation ID if not provided
-    if not conversation_id:
-        conversation_id = start_new_conversation()
-
-    command_history = CommandHistory()
-    # Load specified files if any
-    if files:
-        for file in files:
-            extension = os.path.splitext(file)[1].lower()
-            try:
-                if extension == ".pdf":
-                    content = load_pdf(file)["texts"].iloc[0]
-                elif extension == ".csv":
-                    content = load_csv(file)
-                else:
-                    print(f"Unsupported file type: {file}")
-                    continue
-                loaded_content[file] = content
-                print(f"Loaded content from: {file}")
-            except Exception as e:
-                print(f"Error loading {file}: {str(e)}")
-
-
-    try:
-        while True:
-
-            if loaded_content:
-                context_content = ""
-                for filename, content in loaded_content.items():
-                    retrieved_docs = rag_search(
-                        user_input,
-                        content,
-                    )
-                    if retrieved_docs:
-                        context_content += (
-                            f"\n\nLoaded content from: {filename}\n{content}\n\n"
-                        )
-                if len(context_content) > 0:
-                    user_input += f"""
-                    Here is the loaded content that may be relevant to your query:
-                        {context_content}
-                    Please reference it explicitly in your response and use it for answering.
-                    """
-            if answer_only:
-                response = get_llm_response(
-                user_input, 
-                model = reasoning_model, 
-                provider=reasoning_provider, 
-                messages=messages, 
-                stream=True,
-                )
-                assistant_reply, messages = response['response'], response['messages']
-                assistant_reply = print_and_process_stream_with_markdown(assistant_reply, reasoning_model, reasoning_provider)
-                messages.append({'role':'assistant', 'content':assistant_reply})
-                return enter_reasoning_human_in_the_loop(user_input = None, 
-                                                         messages=messages, 
-                                                         reasoning_model=reasoning_model, 
-                                                         reasoning_provider=reasoning_provider, answer_only=False)
-            else:
-                message= "Think first though and use <think> tags in your chain of thought. Once finished, either answer plainly or write a request for input by beginning with the <request_for_input> tag. and close it with a </request_for_input>"
-                if user_input is None:
-                    user_input = input('🐻‍❄️>')
-                
-                message_id = save_conversation_message(
-                    command_history,
-                    conversation_id,
-                    "user",
-                    user_input,
-                    wd=os.getcwd(),
-                    model=reasoning_model,
-                    provider=reasoning_provider,
-                    npc=npc.name if npc else None,
-                    
-                )                    
-                response = get_llm_response(
-                    user_input+message, 
-                    model = reasoning_model, 
-                    provider=reasoning_provider, 
-                    messages=messages, 
-                    stream=True,
-                )
-
-                assistant_reply, messages = response['response'], response['messages']
-                thoughts = []
-                response_chunks = []
-                in_think_block = False # the thinking chain generated after reasoning
-                
-                thinking = False # the reasoning content 
-                
-
-                for chunk in assistant_reply:       
-                    if thinking:
-                        if not in_think_block:
-                            in_think_block = True
-                    try:
-                        
-                        if reasoning_provider == "ollama":
-                            chunk_content = chunk.get("message", {}).get("content", "")
-                        else:
-                            chunk_content = ''
-                            reasoning_content = ''
-                            for c in chunk.choices:
-                                if hasattr(c.delta, "reasoning_content"):
-                                    
-                                    reasoning_content += c.delta.reasoning_content
-                                    
-                            if reasoning_content:
-                                thinking = True
-                                chunk_content = reasoning_content
-                            chunk_content += "".join(
-                                choice.delta.content
-                                for choice in chunk.choices
-                                if choice.delta.content is not None
-                            )
-                        response_chunks.append(chunk_content)
-                        print(chunk_content, end='')
-                        combined_text = "".join(response_chunks)
-
-                        if in_think_block:
-                            if '</thinking>' in combined_text:
-                                in_think_block = False
-                            thoughts.append(chunk_content)
-                            
-                        if "</request_for_input>" in combined_text:
-                            # Process the LLM's input request
-                            request_text = "".join(thoughts)
-
-                            print("\nPlease provide the requested information: ")
-
-                            user_input = input('🐻‍❄️>')
-
-                            messages.append({"role": "assistant", "content": request_text})
-
-                            print("\n[Continuing with provided information...]\n")
-                            return enter_reasoning_human_in_the_loop( user_input = user_input,
-                                                                     messages=messages, 
-                                                                     reasoning_model=reasoning_model,
-                                                                     reasoning_provider=reasoning_provider,
-                                                                     npc=npc, 
-                                                                     answer_only=True)
-                            
-                        
-                    except KeyboardInterrupt:        
-                        user_interrupt = input("\n[Stream interrupted by user]\n Enter your additional input: ")
-                        
-
-                        # Add the interruption to messages and restart stream
-                        messages.append(
-                            {"role": "user", "content": f"[INTERRUPT] {user_interrupt}"}
-                        )
-                        print(f"\n[Continuing with added context...]\n")        
-                        
-    except KeyboardInterrupt:
-        user_interrupt = input("\n[Stream interrupted by user]\n 🔴🔴🔴🔴\nEnter your additional input: ")
-        
-
-        # Add the interruption to messages and restart stream
-        messages.append(
-            {"role": "user", "content": f"[INTERRUPT] {user_interrupt}"}
-        )
-        print(f"\n[Continuing with added context...]\n")        
-        
-    return {'messages':messages, }
-        
-
-def main():
-    # Example usage
-    import argparse    
-    parser = argparse.ArgumentParser(description="Enter PTI mode for chatting with an LLM")
-    parser.add_argument("--npc", default='~/.npcsh/npc_team/frederic.npc', help="Path to NPC File")    
-    parser.add_argument("--model", default=NPCSH_REASONING_MODEL, help="Model to use")
-    parser.add_argument("--provider", default=NPCSH_REASONING_PROVIDER, help="Provider to use")
-    parser.add_argument("--files", nargs="*", help="Files to load into context")
-    args = parser.parse_args()
-    
-    npc = NPC(file=args.npc)
-    enter_reasoning_human_in_the_loop(
-        messages = [],
-        npc=npc,
-        reasoning_model=args.model,
-        reasoning_provider=args.provider,
-        files=args.files,
-    )
-
-if __name__ == "__main__":
-    main()
-