npcsh 0.1.2__py3-none-any.whl → 1.1.13__py3-none-any.whl

npcsh/npc_team/jinxs/modes/plonk.jinx

@@ -0,0 +1,214 @@
+jinx_name: plonk
+description: Vision-based GUI automation - use vision model to interact with screen elements
+inputs:
+  - task: null
+  - vmodel: null
+  - vprovider: null
+  - max_iterations: 10
+  - debug: true
+
+steps:
+  - name: plonk_execute
+    engine: python
+    code: |
+      import os
+      import time
+      import platform
+      from termcolor import colored
+
+      from npcpy.llm_funcs import get_llm_response
+      from npcpy.data.image import capture_screenshot
+      from npcpy.work.desktop import perform_action
+
+      npc = context.get('npc')
+      messages = context.get('messages', [])
+
+      task = context.get('task')
+      vision_model = context.get('vmodel') or (npc.model if npc else 'gpt-4o')
+      vision_provider = context.get('vprovider') or (npc.provider if npc else 'openai')
+      max_iterations = int(context.get('max_iterations', 10))
+      debug = context.get('debug', True)
+
+      if not task:
+          context['output'] = """Usage: /plonk <task description>
+
+      Options:
+        --vmodel MODEL      Vision model to use (default: gpt-4o)
+        --vprovider PROV    Vision provider (default: openai)
+        --max-iterations N  Max steps (default: 10)
+
+      Example: /plonk Open Firefox and navigate to google.com"""
+          context['messages'] = messages
+          exit()
+
+      print(f"""
+      ██████╗ ██╗      ██████╗ ███╗   ██╗██╗  ██╗
+      ██╔══██╗██║     ██╔═══██╗████╗  ██║██║ ██╔╝
+      ██████╔╝██║     ██║   ██║██╔██╗ ██║█████╔╝
+      ██╔═══╝ ██║     ██║   ██║██║╚██╗██║██╔═██╗
+      ██║     ███████╗╚██████╔╝██║ ╚████║██║  ██╗
+      ╚═╝     ╚══════╝ ╚═════╝ ╚═╝  ╚═══╝╚═╝  ╚═╝
+
+      Vision GUI Automation
+      Task: {task}
+      Model: {vision_model} | Max iterations: {max_iterations}
+      """)
+
+      # System-specific examples
+      system = platform.system()
+      if system == "Windows":
+          app_examples = "start firefox, notepad, calc"
+      elif system == "Darwin":
+          app_examples = "open -a Firefox, open -a TextEdit"
+      else:
+          app_examples = "firefox &, gedit &, gnome-calculator &"
+
+      # Action types
+      ACTION_SCHEMA = {
+          "type": "object",
+          "properties": {
+              "action": {
+                  "type": "string",
+                  "enum": ["click", "type", "key", "launch", "wait", "done", "fail"],
+                  "description": "Action to perform"
+              },
+              "x": {"type": "number", "description": "X coordinate (0-100 percentage)"},
+              "y": {"type": "number", "description": "Y coordinate (0-100 percentage)"},
+              "text": {"type": "string", "description": "Text to type or key to press"},
+              "command": {"type": "string", "description": "Command to launch"},
+              "duration": {"type": "number", "description": "Wait duration in seconds"},
+              "reason": {"type": "string", "description": "Explanation of action"}
+          },
+          "required": ["action", "reason"]
+      }
+
+      click_history = []
+      summary = []
+
+      for iteration in range(max_iterations):
+          print(colored(f"\n--- Iteration {iteration + 1}/{max_iterations} ---", "cyan"))
+
+          # Capture screenshot
+          ss = capture_screenshot()
+          if not ss or 'file_path' not in ss:
+              print(colored("Failed to capture screenshot", "red"))
+              break
+
+          screenshot_path = ss['file_path']
+          if debug:
+              print(colored(f"Screenshot: {screenshot_path}", "gray"))
+
+          # Build context from history
+          history_context = ""
+          if click_history:
+              history_context = f"\nPrevious actions ({len(click_history)}):\n"
+              for i, click in enumerate(click_history[-5:], 1):
+                  history_context += f"  {i}. {click.get('action', 'unknown')} at ({click.get('x', '?')}, {click.get('y', '?')}) - {click.get('reason', '')}\n"
+
+          prompt = f"""You are a GUI automation assistant. Analyze this screenshot and determine the next action to complete the task.
+
+      TASK: {task}
+
+      {history_context}
+
+      Available actions:
+      - click: Click at x,y coordinates (0-100 percentage of screen)
+      - type: Type text
+      - key: Press key (enter, tab, escape, etc.)
+      - launch: Launch application ({app_examples})
+      - wait: Wait for duration seconds
+      - done: Task completed successfully
+      - fail: Task cannot be completed
+
+      Respond with JSON: {{"action": "...", "x": N, "y": N, "text": "...", "command": "...", "duration": N, "reason": "..."}}"""
+
+          try:
+              resp = get_llm_response(
+                  prompt,
+                  model=vision_model,
+                  provider=vision_provider,
+                  images=[screenshot_path],
+                  format="json",
+                  npc=npc
+              )
+
+              action_response = resp.get('response', {})
+              if isinstance(action_response, str):
+                  import json
+                  try:
+                      action_response = json.loads(action_response)
+                  except:
+                      print(colored(f"Invalid JSON response: {action_response[:100]}", "red"))
+                      continue
+
+              action = action_response.get('action', 'fail')
+              reason = action_response.get('reason', 'No reason provided')
+
+              print(colored(f"Action: {action} - {reason}", "yellow"))
+
+              if action == 'done':
+                  print(colored("Task completed successfully!", "green"))
+                  summary.append({"iteration": iteration + 1, "action": "done", "reason": reason})
+                  break
+
+              if action == 'fail':
+                  print(colored(f"Task failed: {reason}", "red"))
+                  summary.append({"iteration": iteration + 1, "action": "fail", "reason": reason})
+                  break
+
+              # Execute action
+              if action == 'click':
+                  x, y = action_response.get('x', 50), action_response.get('y', 50)
+                  perform_action('click', x=x, y=y)
+                  click_history.append({"action": "click", "x": x, "y": y, "reason": reason})
+                  print(colored(f"Clicked at ({x}, {y})", "green"))
+
+              elif action == 'type':
+                  text = action_response.get('text', '')
+                  perform_action('type', text=text)
+                  click_history.append({"action": "type", "text": text[:20], "reason": reason})
+                  print(colored(f"Typed: {text[:30]}...", "green"))
+
+              elif action == 'key':
+                  key = action_response.get('text', 'enter')
+                  perform_action('key', key=key)
+                  click_history.append({"action": "key", "key": key, "reason": reason})
+                  print(colored(f"Pressed key: {key}", "green"))
+
+              elif action == 'launch':
+                  cmd = action_response.get('command', '')
+                  perform_action('launch', command=cmd)
+                  click_history.append({"action": "launch", "command": cmd, "reason": reason})
+                  print(colored(f"Launched: {cmd}", "green"))
+                  time.sleep(2)  # Wait for app to open
+
+              elif action == 'wait':
+                  duration = action_response.get('duration', 1)
+                  time.sleep(duration)
+                  click_history.append({"action": "wait", "duration": duration, "reason": reason})
+                  print(colored(f"Waited {duration}s", "green"))
+
+              summary.append({
+                  "iteration": iteration + 1,
+                  "action": action,
+                  "last_click_coords": f"({click_history[-1].get('x', 'N/A')}, {click_history[-1].get('y', 'N/A')})" if click_history else "N/A",
+                  "reason": reason
+              })
+
+              time.sleep(0.5)  # Brief pause between actions
+
+          except Exception as e:
+              print(colored(f"Error in iteration {iteration + 1}: {e}", "red"))
+              summary.append({"iteration": iteration + 1, "error": str(e)})
+
+      # Generate summary
+      print("\n" + "="*50)
+      print(colored("PLONK SESSION SUMMARY", "cyan", attrs=['bold']))
+      print("="*50)
+      for s in summary:
+          print(f"  Step {s.get('iteration', '?')}: {s.get('action', 'unknown')} - {s.get('reason', s.get('error', ''))[:60]}")
+
+      context['output'] = f"Plonk completed with {len(summary)} actions"
+      context['messages'] = messages
+      context['plonk_summary'] = summary
+      context['click_history'] = click_history

npcsh/npc_team/jinxs/modes/pti.jinx

@@ -0,0 +1,170 @@
+jinx_name: pti
+description: Pardon-The-Interruption - human-in-the-loop reasoning with think tags and interruptible streaming
+npc: frederic
+inputs:
+  - model: null
+  - provider: null
+  - files: null
+  - reasoning_model: null
+
+steps:
+  - name: pti_repl
+    engine: python
+    code: |
+      import os
+      import sys
+      from termcolor import colored
+
+      from npcpy.llm_funcs import get_llm_response
+      from npcpy.npc_sysenv import get_system_message, render_markdown
+      from npcpy.data.load import load_file_contents
+      from npcpy.data.text import rag_search
+
+      npc = context.get('npc')
+      team = context.get('team')
+      messages = context.get('messages', [])
+      files = context.get('files')
+
+      # PTI uses reasoning model for deeper thinking
+      model = context.get('reasoning_model') or context.get('model') or (npc.model if npc else None)
+      provider = context.get('provider') or (npc.provider if npc else None)
+
+      print("""
+      ██████╗ ████████╗██╗
+      ██╔══██╗╚══██╔══╝██║
+      ██████╔╝   ██║   ██║
+      ██╔═══╝    ██║   ██║
+      ██║        ██║   ██║
+      ╚═╝        ╚═╝   ╚═╝
+
+      Pardon-The-Interruption
+      Human-in-the-loop reasoning mode
+      """)
+
+      npc_name = npc.name if npc else "pti"
+      print(f"Entering PTI mode (NPC: {npc_name}). Type '/pq' to exit.")
+      print("  - AI will use <think> tags for step-by-step reasoning")
+      print("  - Use <request_for_input> to pause and ask questions")
+      print("  - Ctrl+C interrupts stream for immediate feedback")
+
+      # Load files if provided
+      loaded_content = {}
+      if files:
+          if isinstance(files, str):
+              files = [f.strip() for f in files.split(',')]
+          for file_path in files:
+              file_path = os.path.expanduser(file_path)
+              if os.path.exists(file_path):
+                  try:
+                      chunks = load_file_contents(file_path)
+                      loaded_content[file_path] = "\n".join(chunks)
+                      print(colored(f"Loaded: {file_path}", "green"))
+                  except Exception as e:
+                      print(colored(f"Error loading {file_path}: {e}", "red"))
+
+      # System message for PTI mode
+      pti_system = """You are an AI assistant in PTI (Pardon-The-Interruption) mode.
+
+      IMPORTANT INSTRUCTIONS:
+      1. Think step-by-step using <think>...</think> tags to show your reasoning
+      2. When you need more information from the user, use <request_for_input>your question</request_for_input>
+      3. Be thorough but concise in your reasoning
+      4. The user can interrupt at any time to provide guidance
+
+      Example:
+      <think>
+      Let me break this down...
+      Step 1: First I need to understand X
+      Step 2: Then consider Y
+      </think>
+
+      <request_for_input>
+      I notice you mentioned Z. Could you clarify what you mean by that?
+      </request_for_input>"""
+
+      if not messages or messages[0].get("role") != "system":
+          messages.insert(0, {"role": "system", "content": pti_system})
+
+      # REPL loop
+      user_input = None
+      while True:
+          try:
+              if not user_input:
+                  prompt_str = f"{npc_name}:pti> "
+                  user_input = input(prompt_str).strip()
+
+              if not user_input:
+                  user_input = None
+                  continue
+
+              if user_input.lower() == "/pq":
+                  print("Exiting PTI mode.")
+                  break
+
+              # Build prompt with file context
+              prompt_for_llm = user_input
+              if loaded_content:
+                  context_str = "\n".join([f"--- {fname} ---\n{content}" for fname, content in loaded_content.items()])
+                  prompt_for_llm += f"\n\nContext:\n{context_str}"
+
+              prompt_for_llm += "\n\nThink step-by-step using <think> tags. Use <request_for_input> when you need clarification."
+
+              messages.append({"role": "user", "content": user_input})
+
+              try:
+                  resp = get_llm_response(
+                      prompt_for_llm,
+                      model=model,
+                      provider=provider,
+                      messages=messages[:-1],  # Don't duplicate the user message
+                      stream=True,
+                      npc=npc
+                  )
+
+                  response_stream = resp.get('response')
+                  full_response = ""
+                  request_found = False
+
+                  # Stream the response
+                  for chunk in response_stream:
+                      chunk_content = ""
+                      if hasattr(chunk, 'choices') and chunk.choices:
+                          delta = chunk.choices[0].delta
+                          if hasattr(delta, 'content') and delta.content:
+                              chunk_content = delta.content
+                      elif isinstance(chunk, dict):
+                          chunk_content = chunk.get("message", {}).get("content", "")
+
+                      if chunk_content:
+                          print(chunk_content, end='', flush=True)
+                          full_response += chunk_content
+
+                          # Check for request_for_input
+                          if "</request_for_input>" in full_response:
+                              request_found = True
+                              break
+
+                  print()  # newline after stream
+
+                  messages.append({"role": "assistant", "content": full_response})
+                  user_input = None  # Reset for next iteration
+
+              except KeyboardInterrupt:
+                  print(colored("\n\n--- Interrupted ---", "yellow"))
+                  interrupt_input = input("Your feedback: ").strip()
+                  if interrupt_input:
+                      user_input = interrupt_input
+                  else:
+                      user_input = None
+                  continue
+
+          except KeyboardInterrupt:
+              print("\nUse '/pq' to exit or continue.")
+              user_input = None
+              continue
+          except EOFError:
+              print("\nExiting PTI mode.")
+              break
+
+      context['output'] = "Exited PTI mode."
+      context['messages'] = messages

npcsh/npc_team/jinxs/modes/spool.jinx

@@ -0,0 +1,161 @@
+jinx_name: spool
+description: Interactive chat mode - simple conversational interface with an NPC
+inputs:
+  - model: null
+  - provider: null
+  - attachments: null
+  - stream: true
+
+steps:
+  - name: spool_repl
+    engine: python
+    code: |
+      import os
+      import sys
+      from termcolor import colored
+
+      from npcpy.llm_funcs import get_llm_response
+      from npcpy.npc_sysenv import get_system_message, render_markdown
+      from npcpy.data.load import load_file_contents
+      from npcpy.data.text import rag_search
+
+      npc = context.get('npc')
+      team = context.get('team')
+      messages = context.get('messages', [])
+      stream = context.get('stream', True)
+      attachments = context.get('attachments')
+
+      # Use NPC's model/provider or fallback
+      model = context.get('model') or (npc.model if npc else None)
+      provider = context.get('provider') or (npc.provider if npc else None)
+
+      # ASCII art
+      print("""
+       _____ ____   ____   ____  _
+      / ___/|  _ \ / __ \ / __ \| |
+      \___ \| |_) | |  | | |  | | |
+       ___) |  __/| |  | | |  | | |___
+      |____/|_|    \____/ \____/|_____|
+      """)
+
+      npc_name = npc.name if npc else "chat"
+      print(f"Entering spool mode (NPC: {npc_name}). Type '/sq' to exit.")
+
+      # Load attachments if provided
+      loaded_chunks = {}
+      if attachments:
+          if isinstance(attachments, str):
+              attachments = [f.strip() for f in attachments.split(',')]
+          for file_path in attachments:
+              file_path = os.path.expanduser(file_path)
+              if os.path.exists(file_path):
+                  try:
+                      chunks = load_file_contents(file_path)
+                      loaded_chunks[file_path] = chunks
+                      print(colored(f"Loaded {len(chunks)} chunks from: {file_path}", "green"))
+                  except Exception as e:
+                      print(colored(f"Error loading {file_path}: {e}", "red"))
+
+      # Ensure system message
+      if not messages or messages[0].get("role") != "system":
+          sys_msg = get_system_message(npc) if npc else "You are a helpful assistant."
+          messages.insert(0, {"role": "system", "content": sys_msg})
+
+      # REPL loop
+      while True:
+          try:
+              prompt_str = f"{npc_name}> "
+              user_input = input(prompt_str).strip()
+
+              if not user_input:
+                  continue
+
+              if user_input.lower() == "/sq":
+                  print("Exiting spool mode.")
+                  break
+
+              # Handle /ots for screenshots inline
+              if user_input.startswith("/ots"):
+                  from npcpy.data.image import capture_screenshot
+                  parts = user_input.split()
+                  image_paths = []
+                  if len(parts) > 1:
+                      for p in parts[1:]:
+                          fp = os.path.expanduser(p)
+                          if os.path.exists(fp):
+                              image_paths.append(fp)
+                  else:
+                      ss = capture_screenshot()
+                      if ss and "file_path" in ss:
+                          image_paths.append(ss["file_path"])
+                          print(colored(f"Screenshot: {ss['filename']}", "green"))
+
+                  if image_paths:
+                      vision_prompt = input("Prompt for image(s): ").strip() or "Describe these images."
+                      resp = get_llm_response(
+                          vision_prompt,
+                          model=npc.vision_model if hasattr(npc, 'vision_model') else model,
+                          provider=npc.vision_provider if hasattr(npc, 'vision_provider') else provider,
+                          messages=messages,
+                          images=image_paths,
+                          stream=stream,
+                          npc=npc
+                      )
+                      messages = resp.get('messages', messages)
+                      render_markdown(str(resp.get('response', '')))
+                  continue
+
+              # Add RAG context if files loaded
+              current_prompt = user_input
+              if loaded_chunks:
+                  context_content = ""
+                  for filename, chunks in loaded_chunks.items():
+                      full_text = "\n".join(chunks)
+                      retrieved = rag_search(user_input, full_text, similarity_threshold=0.3)
+                      if retrieved:
+                          context_content += f"\n\nContext from {filename}:\n{retrieved}\n"
+                  if context_content:
+                      current_prompt += f"\n\n--- Relevant context ---{context_content}"
+
+              # Get response
+              resp = get_llm_response(
+                  current_prompt,
+                  model=model,
+                  provider=provider,
+                  messages=messages,
+                  stream=stream,
+                  npc=npc
+              )
+
+              messages = resp.get('messages', messages)
+              response_text = resp.get('response', '')
+
+              # Handle streaming vs non-streaming
+              if hasattr(response_text, '__iter__') and not isinstance(response_text, str):
+                  full_response = ""
+                  for chunk in response_text:
+                      if hasattr(chunk, 'choices') and chunk.choices:
+                          delta = chunk.choices[0].delta
+                          if hasattr(delta, 'content') and delta.content:
+                              print(delta.content, end='', flush=True)
+                              full_response += delta.content
+                  print()
+              else:
+                  render_markdown(str(response_text))
+
+              # Track usage if available
+              if 'usage' in resp and npc and hasattr(npc, 'shared_context'):
+                  usage = resp['usage']
+                  npc.shared_context['session_input_tokens'] += usage.get('input_tokens', 0)
+                  npc.shared_context['session_output_tokens'] += usage.get('output_tokens', 0)
+                  npc.shared_context['turn_count'] += 1
+
+          except KeyboardInterrupt:
+              print("\nUse '/sq' to exit or continue.")
+              continue
+          except EOFError:
+              print("\nExiting spool mode.")
+              break
+
+      context['output'] = "Exited spool mode."
+      context['messages'] = messages