npcsh 1.1.12__py3-none-any.whl → 1.1.14__py3-none-any.whl

npcsh/npc_team/jinxs/modes/guac.jinx

@@ -1,46 +1,317 @@
-jinx_name: "guac"
-description: "Enter guac mode for plotting and data visualization."
+jinx_name: guac
+description: Python data analysis mode - execute Python code with persistent locals, auto-load files
 inputs:
-  - config_dir: "" # Optional configuration directory.
-  - plots_dir: "" # Optional directory for plots.
-  - refresh_period: 100 # Refresh period for guac mode.
-  - lang: "" # Language setting for guac mode.
+  - model: null
+  - provider: null
+  - plots_dir: null
+
 steps:
-  - name: "enter_guac"
-    engine: "python"
+  - name: guac_repl
+    engine: python
     code: |
       import os
-      from sqlalchemy import create_engine
-      from npcpy.npc_compiler import NPC, Team
-      from npcsh.guac import enter_guac_mode
-      
-      config_dir = context.get('config_dir')
-      plots_dir = context.get('plots_dir')
-      refresh_period = context.get('refresh_period')
-      lang = context.get('lang')
-      output_messages = context.get('messages', [])
-      
-      db_path = os.path.expanduser('~/npcsh_history.db')
-      db_conn = create_engine(f'sqlite:///{db_path}')
-      
-      npc_file = os.path.expanduser('~/.npcsh/guac/npc_team/guac.npc')
-      npc_team_dir = os.path.expanduser('~/.npcsh/guac/npc_team/')
-      
-      # Ensure directories exist for guac NPC/Team
-      os.makedirs(os.path.dirname(npc_file), exist_ok=True)
-      
-      guac_npc = NPC(file=npc_file, db_conn=db_conn)
-      guac_team = Team(npc_team_dir, db_conn=db_conn)
-      
-      enter_guac_mode(
-          npc=guac_npc, 
-          team=guac_team, 
-          config_dir=config_dir, 
-          plots_dir=plots_dir,
-          npc_team_dir=npc_team_dir,
-          refresh_period=int(refresh_period), # Ensure int type
-          lang=lang
-      )
-      
-      context['output'] = 'Exiting Guac Mode'
-      context['messages'] = output_messages
+      import sys
+      import io
+      import re
+      import traceback
+      from pathlib import Path
+      from datetime import datetime
+      from termcolor import colored
+
+      import numpy as np
+      import pandas as pd
+      import matplotlib.pyplot as plt
+
+      from npcpy.llm_funcs import get_llm_response
+      from npcpy.npc_sysenv import render_markdown, get_system_message
+
+      npc = context.get('npc')
+      team = context.get('team')
+      messages = context.get('messages', [])
+      plots_dir = context.get('plots_dir') or os.path.expanduser("~/.npcsh/plots")
+
+      model = context.get('model') or (npc.model if npc else None)
+      provider = context.get('provider') or (npc.provider if npc else None)
+
+      # Use shared_context for persistent Python locals
+      shared_ctx = npc.shared_context if npc and hasattr(npc, 'shared_context') else {}
+      if 'locals' not in shared_ctx:
+          shared_ctx['locals'] = {}
+
+      # Initialize locals with useful imports
+      guac_locals = shared_ctx['locals']
+      guac_locals.update({
+          'np': np,
+          'pd': pd,
+          'plt': plt,
+          'Path': Path,
+          'os': os,
+      })
+
+      # Also store dataframes reference
+      if 'dataframes' not in shared_ctx:
+          shared_ctx['dataframes'] = {}
+      guac_locals['dataframes'] = shared_ctx['dataframes']
+
+      os.makedirs(plots_dir, exist_ok=True)
+
+      print("""
+       ██████╗ ██╗   ██╗ █████╗  ██████╗
+      ██╔════╝ ██║   ██║██╔══██╗██╔════╝
+      ██║  ███╗██║   ██║███████║██║
+      ██║   ██║██║   ██║██╔══██║██║
+      ╚██████╔╝╚██████╔╝██║  ██║╚██████╗
+       ╚═════╝  ╚═════╝ ╚═╝  ╚═╝ ╚═════╝
+      """)
+
+      npc_name = npc.name if npc else "guac"
+      print(f"Entering guac mode (NPC: {npc_name}). Type '/gq' to exit.")
+      print("  - Type Python code directly to execute")
+      print("  - Type natural language to get code suggestions")
+      print("  - Drop file paths to auto-load data")
+      print(f"  - Plots saved to: {plots_dir}")
+
+      def is_python_code(text):
+          """Check if text looks like Python code"""
+          text = text.strip()
+          if not text:
+              return False
+          # Common Python patterns
+          if re.match(r'^(import |from |def |class |if |for |while |with |try:|@|#)', text):
+              return True
+          if re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s*=', text):
+              return True
+          if re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s*\(', text):
+              return True
+          if re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\s*\[', text):
+              return True
+          if text.startswith('print(') or text.startswith('plt.'):
+              return True
+          try:
+              compile(text, "<input>", "exec")
+              return True
+          except SyntaxError:
+              return False
+
+      def execute_python(code_str, locals_dict):
+          """Execute Python code and return output"""
+          output_capture = io.StringIO()
+          old_stdout, old_stderr = sys.stdout, sys.stderr
+
+          try:
+              sys.stdout = output_capture
+              sys.stderr = output_capture
+
+              # Try as expression first (for REPL-like behavior)
+              if '\n' not in code_str.strip():
+                  try:
+                      result = eval(compile(code_str, "<input>", "eval"), locals_dict)
+                      if result is not None:
+                          print(repr(result))
+                      return output_capture.getvalue().strip()
+                  except SyntaxError:
+                      pass
+
+              # Execute as statements
+              exec(compile(code_str, "<input>", "exec"), locals_dict)
+              return output_capture.getvalue().strip()
+
+          except Exception:
+              traceback.print_exc(file=output_capture)
+              return output_capture.getvalue().strip()
+          finally:
+              sys.stdout, sys.stderr = old_stdout, old_stderr
+
+      def auto_load_file(file_path, locals_dict):
+          """Auto-load a file into locals based on extension"""
+          path = Path(file_path).expanduser()
+          if not path.exists():
+              return None
+
+          ext = path.suffix.lower()
+          var_name = f"data_{datetime.now().strftime('%H%M%S')}"
+
+          try:
+              if ext == '.csv':
+                  df = pd.read_csv(path)
+                  locals_dict[var_name] = df
+                  shared_ctx['dataframes'][var_name] = df
+                  return f"Loaded CSV as '{var_name}': {len(df)} rows, {len(df.columns)} columns\nColumns: {list(df.columns)}"
+
+              elif ext in ['.xlsx', '.xls']:
+                  df = pd.read_excel(path)
+                  locals_dict[var_name] = df
+                  shared_ctx['dataframes'][var_name] = df
+                  return f"Loaded Excel as '{var_name}': {len(df)} rows, {len(df.columns)} columns"
+
+              elif ext == '.json':
+                  import json
+                  with open(path) as f:
+                      data = json.load(f)
+                  locals_dict[var_name] = data
+                  return f"Loaded JSON as '{var_name}': {type(data).__name__}"
+
+              elif ext in ['.png', '.jpg', '.jpeg', '.gif']:
+                  from PIL import Image
+                  img = Image.open(path)
+                  arr = np.array(img)
+                  locals_dict[f"{var_name}_img"] = img
+                  locals_dict[f"{var_name}_arr"] = arr
+                  return f"Loaded image as '{var_name}_img' and '{var_name}_arr': {img.size}"
+
+              elif ext == '.npy':
+                  arr = np.load(path)
+                  locals_dict[var_name] = arr
+                  return f"Loaded numpy array as '{var_name}': shape {arr.shape}"
+
+              elif ext in ['.txt', '.md']:
+                  with open(path) as f:
+                      text = f.read()
+                  locals_dict[var_name] = text
+                  return f"Loaded text as '{var_name}': {len(text)} chars"
+
+              else:
+                  with open(path, 'rb') as f:
+                      data = f.read()
+                  locals_dict[var_name] = data
+                  return f"Loaded binary as '{var_name}': {len(data)} bytes"
+
+          except Exception as e:
+              return f"Error loading {path}: {e}"
+
+      def save_current_plot():
+          """Save current matplotlib figure if any"""
+          if plt.get_fignums():
+              timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+              plot_path = os.path.join(plots_dir, f"plot_{timestamp}.png")
+              plt.savefig(plot_path, dpi=150, bbox_inches='tight')
+              plt.close()
+              return plot_path
+          return None
+
+      # Ensure system message for LLM help
+      if not messages or messages[0].get("role") != "system":
+          sys_msg = """You are a Python data analysis assistant. When the user asks questions:
+      1. Generate clean, executable Python code
+      2. Use pandas, numpy, matplotlib as needed
+      3. Reference variables already in the user's session
+      4. Keep code concise and focused"""
+          messages.insert(0, {"role": "system", "content": sys_msg})
+
+      # REPL loop
+      while True:
+          try:
+              prompt_str = f"{npc_name}:guac> "
+              user_input = input(prompt_str).strip()
+
+              if not user_input:
+                  continue
+
+              if user_input.lower() == "/gq":
+                  print("Exiting guac mode.")
+                  break
+
+              # /vars - show current variables
+              if user_input.lower() == "/vars":
+                  print(colored("Current variables:", "cyan"))
+                  for k, v in guac_locals.items():
+                      if not k.startswith('_') and k not in ['np', 'pd', 'plt', 'Path', 'os', 'dataframes']:
+                          vtype = type(v).__name__
+                          if isinstance(v, pd.DataFrame):
+                              print(f"  {k}: DataFrame ({len(v)} rows)")
+                          elif isinstance(v, np.ndarray):
+                              print(f"  {k}: ndarray {v.shape}")
+                          else:
+                              print(f"  {k}: {vtype}")
+                  continue
+
+              # /clear - clear variables
+              if user_input.lower() == "/clear":
+                  keep = {'np', 'pd', 'plt', 'Path', 'os', 'dataframes'}
+                  guac_locals.clear()
+                  guac_locals.update({k: v for k, v in [('np', np), ('pd', pd), ('plt', plt), ('Path', Path), ('os', os), ('dataframes', shared_ctx['dataframes'])]})
+                  print(colored("Variables cleared.", "yellow"))
+                  continue
+
+              # Check if it's a file path (drag & drop)
+              potential_path = user_input.strip("'\"")
+              if os.path.exists(os.path.expanduser(potential_path)):
+                  result = auto_load_file(potential_path, guac_locals)
+                  if result:
+                      print(colored(result, "green"))
+                  continue
+
+              # Check if it's Python code
+              if is_python_code(user_input):
+                  output = execute_python(user_input, guac_locals)
+                  if output:
+                      print(output)
+
+                  # Save any plots
+                  plot_path = save_current_plot()
+                  if plot_path:
+                      print(colored(f"Plot saved: {plot_path}", "green"))
+                  continue
+
+              # Natural language - ask LLM for code
+              # Include current variables in context
+              var_context = "Current variables:\n"
+              for k, v in guac_locals.items():
+                  if not k.startswith('_') and k not in ['np', 'pd', 'plt', 'Path', 'os', 'dataframes']:
+                      if isinstance(v, pd.DataFrame):
+                          var_context += f"  {k}: DataFrame with columns {list(v.columns)}\n"
+                      elif isinstance(v, np.ndarray):
+                          var_context += f"  {k}: ndarray shape {v.shape}\n"
+                      else:
+                          var_context += f"  {k}: {type(v).__name__}\n"
+
+              prompt = f"{var_context}\nUser request: {user_input}\n\nGenerate Python code to accomplish this. Return ONLY the code, no explanation."
+
+              resp = get_llm_response(
+                  prompt,
+                  model=model,
+                  provider=provider,
+                  messages=messages,
+                  npc=npc
+              )
+
+              messages = resp.get('messages', messages)
+              code_response = str(resp.get('response', ''))
+
+              # Extract code from response (strip markdown if present)
+              code = code_response
+              if '```python' in code:
+                  code = code.split('```python')[1].split('```')[0]
+              elif '```' in code:
+                  code = code.split('```')[1].split('```')[0]
+              code = code.strip()
+
+              if code:
+                  print(colored("Generated code:", "cyan"))
+                  print(code)
+                  confirm = input("Execute? [Y/n/e(dit)]: ").strip().lower()
+
+                  if confirm in ['', 'y', 'yes']:
+                      output = execute_python(code, guac_locals)
+                      if output:
+                          print(output)
+                      plot_path = save_current_plot()
+                      if plot_path:
+                          print(colored(f"Plot saved: {plot_path}", "green"))
+
+                  elif confirm == 'e':
+                      edited = input("Enter modified code: ").strip()
+                      if edited:
+                          output = execute_python(edited, guac_locals)
+                          if output:
+                              print(output)
+
+          except KeyboardInterrupt:
+              print("\nUse '/gq' to exit or continue.")
+              continue
+          except EOFError:
+              print("\nExiting guac mode.")
+              break
+
+      context['output'] = "Exited guac mode."
+      context['messages'] = messages

npcsh/npc_team/jinxs/modes/plonk.jinx

@@ -1,53 +1,214 @@
-jinx_name: "plonk"
-description: "Use vision model to interact with GUI. Usage: /plonk <task description>"
+jinx_name: plonk
+description: Vision-based GUI automation - use vision model to interact with screen elements
 inputs:
-  - task_description: "" # Required task description for GUI interaction.
-  - vmodel: "" # Vision model to use. Defaults to NPCSH_VISION_MODEL or NPC's model.
-  - vprovider: "" # Vision model provider. Defaults to NPCSH_VISION_PROVIDER or NPC's provider.
+  - task: null
+  - vmodel: null
+  - vprovider: null
+  - max_iterations: 10
+  - debug: true
+
 steps:
-  - name: "execute_plonk"
-    engine: "python"
+  - name: plonk_execute
+    engine: python
     code: |
-      import traceback
-      from npcsh.plonk import execute_plonk_command, format_plonk_summary
-      # Assuming NPCSH_VISION_MODEL and NPCSH_VISION_PROVIDER are accessible
-      
-      task_description = context.get('task_description')
-      vision_model = context.get('vmodel')
-      vision_provider = context.get('vprovider')
-      plonk_context = context.get('plonk_context') # Passed from original context
-      current_npc = context.get('npc')
-      output_messages = context.get('messages', [])
-      
-      if not task_description or not task_description.strip():
-          context['output'] = "Usage: /plonk <task_description> [--vmodel model_name] [--vprovider provider_name]"
-          context['messages'] = output_messages
+      import os
+      import time
+      import platform
+      from termcolor import colored
+
+      from npcpy.llm_funcs import get_llm_response
+      from npcpy.data.image import capture_screenshot
+      from npcpy.work.desktop import perform_action
+
+      npc = context.get('npc')
+      messages = context.get('messages', [])
+
+      task = context.get('task')
+      vision_model = context.get('vmodel') or (npc.model if npc else 'gpt-4o')
+      vision_provider = context.get('vprovider') or (npc.provider if npc else 'openai')
+      max_iterations = int(context.get('max_iterations', 10))
+      debug = context.get('debug', True)
+
+      if not task:
+          context['output'] = """Usage: /plonk <task description>
+
+      Options:
+        --vmodel MODEL      Vision model to use (default: gpt-4o)
+        --vprovider PROV    Vision provider (default: openai)
+        --max-iterations N  Max steps (default: 10)
+
+      Example: /plonk Open Firefox and navigate to google.com"""
+          context['messages'] = messages
           exit()
 
-      # Fallback for model/provider if not explicitly set in Jinx inputs
-      if not vision_model and current_npc and current_npc.model:
-          vision_model = current_npc.model
-      if not vision_provider and current_npc and current_npc.provider:
-          vision_provider = current_npc.provider
-      
-      try:
-          summary_data = execute_plonk_command(
-              request=task_description,
-              model=vision_model,
-              provider=vision_provider,
-              npc=current_npc,
-              plonk_context=plonk_context,
-              debug=True # Assuming debug is often desired for plonk
-          )        
-          
-          if summary_data and isinstance(summary_data, list):
-              output_report = format_plonk_summary(summary_data)
-              context['output'] = output_report
-          else:
-              context['output'] = "Plonk command did not complete within the maximum number of iterations."
-
-      except Exception as e:
-          traceback.print_exc()
-          context['output'] = f"Error executing plonk command: {e}"
-      
-      context['messages'] = output_messages
+      print(f"""
+      ██████╗ ██╗      ██████╗ ███╗   ██╗██╗  ██╗
+      ██╔══██╗██║     ██╔═══██╗████╗  ██║██║ ██╔╝
+      ██████╔╝██║     ██║   ██║██╔██╗ ██║█████╔╝
+      ██╔═══╝ ██║     ██║   ██║██║╚██╗██║██╔═██╗
+      ██║     ███████╗╚██████╔╝██║ ╚████║██║  ██╗
+      ╚═╝     ╚══════╝ ╚═════╝ ╚═╝  ╚═══╝╚═╝  ╚═╝
+
+      Vision GUI Automation
+      Task: {task}
+      Model: {vision_model} | Max iterations: {max_iterations}
+      """)
+
+      # System-specific examples
+      system = platform.system()
+      if system == "Windows":
+          app_examples = "start firefox, notepad, calc"
+      elif system == "Darwin":
+          app_examples = "open -a Firefox, open -a TextEdit"
+      else:
+          app_examples = "firefox &, gedit &, gnome-calculator &"
+
+      # Action types
+      ACTION_SCHEMA = {
+          "type": "object",
+          "properties": {
+              "action": {
+                  "type": "string",
+                  "enum": ["click", "type", "key", "launch", "wait", "done", "fail"],
+                  "description": "Action to perform"
+              },
+              "x": {"type": "number", "description": "X coordinate (0-100 percentage)"},
+              "y": {"type": "number", "description": "Y coordinate (0-100 percentage)"},
+              "text": {"type": "string", "description": "Text to type or key to press"},
+              "command": {"type": "string", "description": "Command to launch"},
+              "duration": {"type": "number", "description": "Wait duration in seconds"},
+              "reason": {"type": "string", "description": "Explanation of action"}
+          },
+          "required": ["action", "reason"]
+      }
+
+      click_history = []
+      summary = []
+
+      for iteration in range(max_iterations):
+          print(colored(f"\n--- Iteration {iteration + 1}/{max_iterations} ---", "cyan"))
+
+          # Capture screenshot
+          ss = capture_screenshot()
+          if not ss or 'file_path' not in ss:
+              print(colored("Failed to capture screenshot", "red"))
+              break
+
+          screenshot_path = ss['file_path']
+          if debug:
+              print(colored(f"Screenshot: {screenshot_path}", "gray"))
+
+          # Build context from history
+          history_context = ""
+          if click_history:
+              history_context = f"\nPrevious actions ({len(click_history)}):\n"
+              for i, click in enumerate(click_history[-5:], 1):
+                  history_context += f"  {i}. {click.get('action', 'unknown')} at ({click.get('x', '?')}, {click.get('y', '?')}) - {click.get('reason', '')}\n"
+
+          prompt = f"""You are a GUI automation assistant. Analyze this screenshot and determine the next action to complete the task.
+
+      TASK: {task}
+
+      {history_context}
+
+      Available actions:
+      - click: Click at x,y coordinates (0-100 percentage of screen)
+      - type: Type text
+      - key: Press key (enter, tab, escape, etc.)
+      - launch: Launch application ({app_examples})
+      - wait: Wait for duration seconds
+      - done: Task completed successfully
+      - fail: Task cannot be completed
+
+      Respond with JSON: {{"action": "...", "x": N, "y": N, "text": "...", "command": "...", "duration": N, "reason": "..."}}"""
+
+          try:
+              resp = get_llm_response(
+                  prompt,
+                  model=vision_model,
+                  provider=vision_provider,
+                  images=[screenshot_path],
+                  format="json",
+                  npc=npc
+              )
+
+              action_response = resp.get('response', {})
+              if isinstance(action_response, str):
+                  import json
+                  try:
+                      action_response = json.loads(action_response)
+                  except:
+                      print(colored(f"Invalid JSON response: {action_response[:100]}", "red"))
+                      continue
+
+              action = action_response.get('action', 'fail')
+              reason = action_response.get('reason', 'No reason provided')
+
+              print(colored(f"Action: {action} - {reason}", "yellow"))
+
+              if action == 'done':
+                  print(colored("Task completed successfully!", "green"))
+                  summary.append({"iteration": iteration + 1, "action": "done", "reason": reason})
+                  break
+
+              if action == 'fail':
+                  print(colored(f"Task failed: {reason}", "red"))
+                  summary.append({"iteration": iteration + 1, "action": "fail", "reason": reason})
+                  break
+
+              # Execute action
+              if action == 'click':
+                  x, y = action_response.get('x', 50), action_response.get('y', 50)
+                  perform_action('click', x=x, y=y)
+                  click_history.append({"action": "click", "x": x, "y": y, "reason": reason})
+                  print(colored(f"Clicked at ({x}, {y})", "green"))
+
+              elif action == 'type':
+                  text = action_response.get('text', '')
+                  perform_action('type', text=text)
+                  click_history.append({"action": "type", "text": text[:20], "reason": reason})
+                  print(colored(f"Typed: {text[:30]}...", "green"))
+
+              elif action == 'key':
+                  key = action_response.get('text', 'enter')
+                  perform_action('key', key=key)
+                  click_history.append({"action": "key", "key": key, "reason": reason})
+                  print(colored(f"Pressed key: {key}", "green"))
+
+              elif action == 'launch':
+                  cmd = action_response.get('command', '')
+                  perform_action('launch', command=cmd)
+                  click_history.append({"action": "launch", "command": cmd, "reason": reason})
+                  print(colored(f"Launched: {cmd}", "green"))
+                  time.sleep(2)  # Wait for app to open
+
+              elif action == 'wait':
+                  duration = action_response.get('duration', 1)
+                  time.sleep(duration)
+                  click_history.append({"action": "wait", "duration": duration, "reason": reason})
+                  print(colored(f"Waited {duration}s", "green"))
+
+              summary.append({
+                  "iteration": iteration + 1,
+                  "action": action,
+                  "last_click_coords": f"({click_history[-1].get('x', 'N/A')}, {click_history[-1].get('y', 'N/A')})" if click_history else "N/A",
+                  "reason": reason
+              })
+
+              time.sleep(0.5)  # Brief pause between actions
+
+          except Exception as e:
+              print(colored(f"Error in iteration {iteration + 1}: {e}", "red"))
+              summary.append({"iteration": iteration + 1, "error": str(e)})
+
+      # Generate summary
+      print("\n" + "="*50)
+      print(colored("PLONK SESSION SUMMARY", "cyan", attrs=['bold']))
+      print("="*50)
+      for s in summary:
+          print(f"  Step {s.get('iteration', '?')}: {s.get('action', 'unknown')} - {s.get('reason', s.get('error', ''))[:60]}")
+
+      context['output'] = f"Plonk completed with {len(summary)} actions"
+      context['messages'] = messages
+      context['plonk_summary'] = summary
+      context['click_history'] = click_history