npcsh 1.1.21__py3-none-any.whl → 1.1.23__py3-none-any.whl

{npcsh-1.1.21.data → npcsh-1.1.23.data}/data/npcsh/npc_team/plonk.jinx

@@ -24,6 +24,25 @@ steps:
 
       from npcpy.llm_funcs import get_llm_response
 
+      # Helper to log jinx executions to DB
+      def _log_jinx(trigger_id, npc_name, inputs, output, status="success", error_msg=None):
+          try:
+              if state and hasattr(state, 'command_history') and state.command_history is not None and hasattr(state.command_history, 'save_jinx_execution'):
+                  _conv_id = getattr(state, 'conversation_id', None) or ''
+                  state.command_history.save_jinx_execution(
+                      triggering_message_id=f"{_conv_id}-{trigger_id}",
+                      conversation_id=_conv_id,
+                      npc_name=npc_name,
+                      jinx_name="plonk",
+                      jinx_inputs=inputs,
+                      jinx_output=str(output) if output else "",
+                      status=status,
+                      team_name=state.team.name if state and hasattr(state, 'team') and state.team else None,
+                      error_message=error_msg,
+                  )
+          except Exception:
+              pass
+
       try:
           from npcpy.data.image import capture_screenshot
           from npcpy.work.desktop import perform_action
@@ -123,7 +142,7 @@ steps:
           render_screen()
 
           try:
-              ss = capture_screenshot()
+              ss = capture_screenshot(full=True)
               if not ss or 'file_path' not in ss:
                   task['actions'].append({"action": "fail", "reason": "Screenshot failed"})
                   task['status'] = 'failed'
@@ -136,23 +155,43 @@ steps:
               history_context = ""
               if task['actions']:
                   history_context = "\nPrevious actions:\n"
-                  for i, act in enumerate(task['actions'][-5:], 1):
+                  for i, act in enumerate(task['actions'][-8:], 1):
                       history_context += "  " + str(i) + ". " + act.get('action', '?')
                       if act.get('x'):
                           history_context += " at (" + str(act.get('x', '?')) + ", " + str(act.get('y', '?')) + ")"
+                      if act.get('text'):
+                          history_context += ' "' + act.get('text', '')[:50] + '"'
+                      if act.get('command'):
+                          history_context += ' [' + act.get('command', '')[:30] + ']'
                       history_context += " - " + act.get('reason', '') + "\n"
+                  # Detect repetition: if last 3 actions are the same type, warn
+                  recent = task['actions'][-3:]
+                  if len(recent) == 3 and all(a.get('action') == recent[0].get('action') for a in recent):
+                      history_context += "\n*** WARNING: You have repeated '" + recent[0].get('action', '?') + "' 3 times. "
+                      history_context += "This is NOT working. You MUST try a completely different approach. ***\n"
 
               prompt = "You are a GUI automation assistant. Analyze this screenshot and determine the next action.\n\n"
+              prompt += "CRITICAL RULES:\n"
+              prompt += "1. VERIFY: Check the screenshot BEFORE acting. Does it show the result of your last action? If not, wait.\n"
+              prompt += "2. NO REPEATS: If an action failed or had no effect, try a DIFFERENT approach. Never repeat the same action.\n"
+              prompt += "3. FULL TEXT: Always type the COMPLETE text in a single 'type' action. Never truncate or split text.\n"
+              prompt += "4. FOCUS FIRST: Before typing, make sure the target field is focused (click it or use keyboard shortcut).\n\n"
+              prompt += "KEYBOARD SHORTCUTS (use 'key' action with these):\n"
+              prompt += "- ctrl+l or F6: Focus browser address/search bar\n"
+              prompt += "- ctrl+a: Select all text in current field\n"
+              prompt += "- ctrl+t: New browser tab\n"
+              prompt += "- tab/shift+tab: Move between form fields\n"
+              prompt += "- escape: Close popups, cancel autocomplete\n\n"
               prompt += "TASK: " + task['text'] + "\n"
               prompt += history_context + "\n"
               prompt += "Available actions:\n"
-              prompt += "- click: Click at x,y coordinates (0-100 percentage of screen)\n"
-              prompt += "- type: Type text (use 'text' field)\n"
-              prompt += "- key: Press key like enter, tab, escape (use 'text' field)\n"
-              prompt += "- launch: Launch application (use 'command' field, e.g. " + app_examples + ")\n"
-              prompt += "- wait: Wait for 'duration' seconds\n"
-              prompt += "- done: Task completed successfully\n"
-              prompt += "- fail: Task cannot be completed\n\n"
+              prompt += "- click: Click at x,y (0-100 % of screen). Fields: x, y\n"
+              prompt += "- type: Type text into focused field. Fields: text (MUST be complete text)\n"
+              prompt += "- key: Press key(s). Fields: text (e.g. 'enter', 'ctrl+l', 'tab', 'escape')\n"
+              prompt += "- launch: Launch application. Fields: command (e.g. " + app_examples + ")\n"
+              prompt += "- wait: Wait seconds. Fields: duration\n"
+              prompt += "- done: Task completed\n"
+              prompt += "- fail: Task impossible\n\n"
               prompt += "Respond with JSON, e.g.: " + json_schema_example
 
               ui.status = "Thinking... (iter " + str(ui.iteration) + "/" + str(ui.max_iter) + ")"
@@ -173,21 +212,38 @@ steps:
                       action_response = json.loads(action_response)
                   except:
                       task['actions'].append({"action": "error", "reason": "Invalid JSON from model"})
+                      _log_jinx(f"plonk-task-{ui.current_task}-iter-{ui.iteration}",
+                                npc.name if npc and hasattr(npc, 'name') else "plonk",
+                                {"task": task['text'], "iteration": ui.iteration, "type": "analysis"},
+                                str(resp.get('response', '')), status="error", error_msg="Invalid JSON from model")
                       ui.status = "Bad response, retrying..."
                       return
 
               action = action_response.get('action', 'fail')
               reason = action_response.get('reason', '')
 
+              _npc_name = npc.name if npc and hasattr(npc, 'name') else "plonk"
+              _log_jinx(f"plonk-task-{ui.current_task}-iter-{ui.iteration}",
+                        _npc_name,
+                        {"task": task['text'], "iteration": ui.iteration, "screenshot": screenshot_path,
+                         "type": "analysis"},
+                        json.dumps(action_response))
+
               if action == 'done':
                   task['status'] = 'done'
                   task['actions'].append({"action": "done", "reason": reason})
+                  _log_jinx(f"plonk-task-{ui.current_task}-done", _npc_name,
+                            {"task": task['text'], "type": "task_complete", "total_actions": len(task['actions'])},
+                            reason)
                   advance_to_next_task()
                   return
 
               if action == 'fail':
                   task['status'] = 'failed'
                   task['actions'].append({"action": "fail", "reason": reason})
+                  _log_jinx(f"plonk-task-{ui.current_task}-fail", _npc_name,
+                            {"task": task['text'], "type": "task_failed", "total_actions": len(task['actions'])},
+                            reason, status="error", error_msg=reason)
                   advance_to_next_task()
                   return
 
@@ -196,20 +252,25 @@ steps:
 
               if action == 'click':
                   x, y = action_response.get('x', 50), action_response.get('y', 50)
-                  perform_action('click', x=x, y=y)
+                  perform_action({"type": "click", "x": x, "y": y})
                   act_record['x'] = x
                   act_record['y'] = y
               elif action == 'type':
                   txt = action_response.get('text', '')
-                  perform_action('type', text=txt)
+                  perform_action({"type": "type", "text": txt})
                   act_record['text'] = txt
               elif action == 'key':
                   key = action_response.get('text', 'enter')
-                  perform_action('key', key=key)
+                  # Detect combo keys like ctrl+l, ctrl+a, shift+tab
+                  if '+' in key:
+                      parts = [k.strip() for k in key.split('+')]
+                      perform_action({"type": "hotkey", "keys": parts})
+                  else:
+                      perform_action({"type": "key", "keys": key})
                   act_record['key'] = key
               elif action == 'launch':
                   cmd = action_response.get('command', '')
-                  perform_action('launch', command=cmd)
+                  perform_action({"type": "shell", "command": cmd})
                   act_record['command'] = cmd
                   time.sleep(2)
               elif action == 'wait':
@@ -219,7 +280,13 @@ steps:
 
               task['actions'].append(act_record)
               ui.status = action + " - " + reason[:40]
-              time.sleep(0.3)
+              # Wait for UI to settle after state-changing actions
+              if action in ('key', 'click'):
+                  time.sleep(2.0)
+              elif action == 'type':
+                  time.sleep(0.5)
+              else:
+                  time.sleep(0.3)
 
               if ui.mode == 'step':
                   ui.mode = 'paused'
@@ -227,6 +294,10 @@ steps:
 
           except Exception as e:
               task['actions'].append({"action": "error", "reason": str(e)})
+              _log_jinx(f"plonk-task-{ui.current_task}-iter-{ui.iteration}-error",
+                        npc.name if npc and hasattr(npc, 'name') else "plonk",
+                        {"task": task['text'] if task else "unknown", "iteration": ui.iteration, "type": "error"},
+                        str(e), status="error", error_msg=str(e))
               ui.status = "Error: " + str(e)[:40]
 
       def advance_to_next_task():
@@ -363,7 +434,7 @@ steps:
                   if a.get('x') is not None:
                       coords = "(" + str(a.get('x', '')) + "," + str(a.get('y', '')) + ") "
                   elif a.get('text'):
-                      coords = '"' + str(a['text'])[:15] + '" '
+                      coords = '"' + str(a['text'])[:40] + '" '
                   elif a.get('key'):
                       coords = '[' + str(a['key']) + '] '
                   elif a.get('command'):

{npcsh-1.1.21.data → npcsh-1.1.23.data}/data/npcsh/npc_team/plonk.npc

@@ -11,15 +11,11 @@ colors:
   top: "34,139,34"
   bottom: "139,69,19"
 primary_directive: |
-  You are plonk, the browser and GUI automation specialist.
-
-  Browser tools: open_browser, browser_action, browser_screenshot, close_browser
-
-  browser_action actions: click, type, type_and_enter, select, wait, scroll, get_text, get_page, get_elements, press_key
-
-  Use get_elements to discover selectors on the page. Use xpath:// prefix for XPath selectors.
-
-  Desktop tools: screenshot, click, type_text, key_press, launch_app, wait
+  You are plonk, the automation specialist for browser and desktop.
+  Browser: open_browser, browser_action, browser_screenshot, close_browser.
+  browser_action: click, type, type_and_enter, select, wait, scroll, get_text, get_page, get_elements, press_key.
+  Desktop: screenshot, click, type_text, key_press, launch_app, wait.
+  Use get_elements to find selectors. Use xpath:// for XPath. Screenshot after each step to verify.
 jinxs:
   - lib/browser/*
   - lib/computer_use/*

npcsh-1.1.23.data/data/npcsh/npc_team/roll.jinx

@@ -0,0 +1,378 @@
+jinx_name: "roll"
+description: "Video creation studio TUI - generate and manage videos with parameter controls"
+interactive: true
+inputs:
+  - prompt: null
+  - model: null
+  - provider: null
+  - output_path: null
+  - num_frames: null
+  - width: null
+  - height: null
+steps:
+  - name: "roll_tui"
+    engine: "python"
+    code: |
+      import os
+      import sys
+      import tty
+      import termios
+      import select
+      from datetime import datetime
+
+      from npcpy.llm_funcs import gen_video
+
+      npc = context.get('npc')
+      if isinstance(npc, str):
+          npc = None
+
+      # ========== State ==========
+      class RollState:
+          def __init__(self):
+              self.prompt = ""
+              self.model = ""
+              self.provider = ""
+              self.width_val = 256
+              self.height_val = 256
+              self.num_frames = 125
+              self.output_path = ""
+              # UI state
+              self.params = ['prompt', 'model', 'provider', 'width', 'height', 'num_frames', 'output']
+              self.sel = 0
+              self.scroll = 0
+              self.mode = 'params'  # params, editing, gallery, generating
+              self.edit_buf = ""
+              self.edit_cursor = 0
+              self.status = "Ready"
+              self.gallery = []  # [{"path": str, "prompt": str, "timestamp": str}]
+              self.gallery_sel = 0
+              self.gallery_scroll = 0
+
+      ui = RollState()
+
+      # Load from context
+      ui.prompt = str(context.get('prompt') or '')
+      ui.model = str(context.get('model') or os.getenv('NPCSH_VIDEO_GEN_MODEL', ''))
+      ui.provider = str(context.get('provider') or os.getenv('NPCSH_VIDEO_GEN_PROVIDER', ''))
+      if not ui.model and npc and hasattr(npc, 'model') and npc.model:
+          ui.model = npc.model
+      if not ui.provider and npc and hasattr(npc, 'provider') and npc.provider:
+          ui.provider = npc.provider
+      if not ui.model:
+          ui.model = "stable-video-diffusion"
+      if not ui.provider:
+          ui.provider = "diffusers"
+      try:
+          ui.width_val = int(context.get('width') or 256)
+      except:
+          pass
+      try:
+          ui.height_val = int(context.get('height') or 256)
+      except:
+          pass
+      try:
+          ui.num_frames = int(context.get('num_frames') or 125)
+      except:
+          pass
+      ui.output_path = str(context.get('output_path') or '')
+
+      # Load existing gallery
+      vid_dir = os.path.expanduser("~/.npcsh/videos/")
+      if os.path.isdir(vid_dir):
+          for f in sorted(os.listdir(vid_dir), reverse=True)[:50]:
+              if f.lower().endswith(('.mp4', '.webm', '.avi', '.mov', '.mkv')):
+                  ui.gallery.append({"path": os.path.join(vid_dir, f), "prompt": "", "timestamp": f})
+
+      def get_size():
+          try:
+              s = os.get_terminal_size()
+              return s.columns, s.lines
+          except:
+              return 80, 24
+
+      def get_param_value(idx):
+          p = ui.params[idx]
+          if p == 'prompt': return ui.prompt
+          if p == 'model': return ui.model or '(default)'
+          if p == 'provider': return ui.provider or '(default)'
+          if p == 'width': return str(ui.width_val)
+          if p == 'height': return str(ui.height_val)
+          if p == 'num_frames': return str(ui.num_frames)
+          if p == 'output': return ui.output_path or '(auto)'
+          return ''
+
+      def set_param_value(idx, val):
+          p = ui.params[idx]
+          if p == 'prompt': ui.prompt = val
+          elif p == 'model': ui.model = val
+          elif p == 'provider': ui.provider = val
+          elif p == 'width':
+              try: ui.width_val = int(val)
+              except: pass
+          elif p == 'height':
+              try: ui.height_val = int(val)
+              except: pass
+          elif p == 'num_frames':
+              try: ui.num_frames = max(1, int(val))
+              except: pass
+          elif p == 'output': ui.output_path = val
+
+      def generate_videos():
+          ui.mode = 'generating'
+          ui.status = "Generating..."
+          render_screen()
+
+          if not ui.prompt:
+              ui.status = "Error: No prompt provided"
+              ui.mode = 'params'
+              return
+
+          try:
+              if ui.output_path and ui.output_path.strip():
+                  out_file = os.path.expanduser(ui.output_path)
+              else:
+                  os.makedirs(vid_dir, exist_ok=True)
+                  out_file = os.path.join(vid_dir, "roll_" + datetime.now().strftime('%Y%m%d_%H%M%S') + ".mp4")
+
+              result = gen_video(
+                  prompt=ui.prompt,
+                  model=ui.model or None,
+                  provider=ui.provider or None,
+                  npc=npc,
+                  num_frames=ui.num_frames,
+                  width=ui.width_val,
+                  height=ui.height_val,
+                  output_path=out_file,
+                  **context.get('api_kwargs', {})
+              )
+
+              if isinstance(result, dict):
+                  msg = result.get('output', 'Video generated.')
+              else:
+                  msg = str(result)
+
+              ui.gallery.insert(0, {"path": out_file, "prompt": ui.prompt, "timestamp": os.path.basename(out_file)})
+              ui.status = "Generated: " + os.path.basename(out_file)
+          except Exception as e:
+              ui.status = "Error: " + str(e)[:60]
+
+          ui.mode = 'params'
+
+      # ========== Rendering ==========
+      def render_screen():
+          width, height = get_size()
+          out = []
+          out.append("\033[H")
+
+          header = " ROLL - Video Creation Studio "
+          out.append("\033[1;1H\033[7;1m" + header.ljust(width) + "\033[0m")
+
+          if ui.mode in ('params', 'editing', 'generating'):
+              # Parameter panel
+              out.append("\033[3;1H\033[36;1m Parameters \033[90m" + ("-" * (width - 13)) + "\033[0m")
+
+              for i, p in enumerate(ui.params):
+                  row = 4 + i
+                  out.append("\033[" + str(row) + ";1H\033[K")
+                  label = p.capitalize() + ":"
+                  val = get_param_value(i)
+
+                  if ui.mode == 'editing' and i == ui.sel:
+                      line = "  " + label.ljust(14) + "\033[7m " + ui.edit_buf + " \033[0m"
+                  else:
+                      line = "  " + label.ljust(14) + val[:width - 18]
+
+                  if i == ui.sel and ui.mode != 'editing':
+                      out.append("\033[7m>" + line + "\033[0m")
+                  else:
+                      out.append(" " + line)
+
+              # Gallery preview
+              gallery_row = 4 + len(ui.params) + 1
+              out.append("\033[" + str(gallery_row) + ";1H\033[33;1m Gallery (" + str(len(ui.gallery)) + ") \033[90m" + ("-" * (width - 20)) + "\033[0m")
+
+              gallery_h = height - gallery_row - 4
+              for i in range(max(0, gallery_h)):
+                  idx = ui.gallery_scroll + i
+                  row = gallery_row + 1 + i
+                  out.append("\033[" + str(row) + ";1H\033[K")
+                  if idx >= len(ui.gallery):
+                      continue
+                  g = ui.gallery[idx]
+                  fname = os.path.basename(g['path'])[:width - 6]
+                  out.append("  " + fname)
+
+          elif ui.mode == 'gallery':
+              out.append("\033[3;1H\033[33;1m Gallery (" + str(len(ui.gallery)) + ") \033[90m" + ("-" * (width - 20)) + "\033[0m")
+
+              gallery_h = height - 6
+              for i in range(gallery_h):
+                  idx = ui.gallery_scroll + i
+                  row = 4 + i
+                  out.append("\033[" + str(row) + ";1H\033[K")
+                  if idx >= len(ui.gallery):
+                      continue
+                  g = ui.gallery[idx]
+                  fname = os.path.basename(g['path'])
+                  if idx == ui.gallery_sel:
+                      out.append("\033[7m> " + fname[:width-4] + "\033[0m")
+                  else:
+                      out.append("  " + fname[:width-4])
+
+          # Status + footer
+          out.append("\033[" + str(height-2) + ";1H\033[K\033[90m" + ("-" * width) + "\033[0m")
+          out.append("\033[" + str(height-1) + ";1H\033[K " + ui.status[:width-2])
+
+          if ui.mode == 'editing':
+              footer = " Type value, Enter:Confirm  Esc:Cancel "
+          elif ui.mode == 'gallery':
+              footer = " j/k:Nav  o:Open  b:Back  q:Quit "
+          elif ui.mode == 'generating':
+              footer = " Generating... "
+          else:
+              footer = " j/k:Nav  e:Edit  Enter:Generate  g:Gallery  q:Quit "
+          out.append("\033[" + str(height) + ";1H\033[K\033[7m" + footer.ljust(width) + "\033[0m")
+
+          sys.stdout.write(''.join(out))
+          sys.stdout.flush()
+
+      # ========== Input Handling ==========
+      def handle_input(c, fd):
+          if ui.mode == 'editing':
+              return handle_edit(c, fd)
+          if ui.mode == 'gallery':
+              return handle_gallery(c, fd)
+
+          if c == '\x1b':
+              if select.select([fd], [], [], 0.05)[0]:
+                  c2 = os.read(fd, 1).decode('latin-1')
+                  if c2 == '[':
+                      c3 = os.read(fd, 1).decode('latin-1')
+                      if c3 == 'A':
+                          ui.sel = max(0, ui.sel - 1)
+                      elif c3 == 'B':
+                          ui.sel = min(len(ui.params) - 1, ui.sel + 1)
+              return True
+
+          if c == 'q':
+              return False
+          elif c == 'j':
+              ui.sel = min(len(ui.params) - 1, ui.sel + 1)
+          elif c == 'k':
+              ui.sel = max(0, ui.sel - 1)
+          elif c == 'e' or c in ('\r', '\n'):
+              if c in ('\r', '\n') and ui.sel == 0 and ui.prompt:
+                  # Enter on prompt with existing prompt = generate
+                  generate_videos()
+              else:
+                  ui.mode = 'editing'
+                  ui.edit_buf = get_param_value(ui.sel)
+                  if ui.edit_buf in ('(default)', '(auto)'):
+                      ui.edit_buf = ""
+                  ui.edit_cursor = len(ui.edit_buf)
+          elif c == 'g':
+              ui.mode = 'gallery'
+              ui.gallery_sel = 0
+              ui.gallery_scroll = 0
+          elif c == 'G':
+              generate_videos()
+
+          return True
+
+      def handle_edit(c, fd):
+          if c == '\x1b':
+              if select.select([fd], [], [], 0.05)[0]:
+                  os.read(fd, 2)
+              ui.mode = 'params'
+              return True
+
+          if c in ('\r', '\n'):
+              set_param_value(ui.sel, ui.edit_buf)
+              ui.mode = 'params'
+              return True
+
+          if c == '\x7f' or c == '\x08':
+              if ui.edit_cursor > 0:
+                  ui.edit_buf = ui.edit_buf[:ui.edit_cursor-1] + ui.edit_buf[ui.edit_cursor:]
+                  ui.edit_cursor -= 1
+          elif c >= ' ' and c <= '~':
+              ui.edit_buf = ui.edit_buf[:ui.edit_cursor] + c + ui.edit_buf[ui.edit_cursor:]
+              ui.edit_cursor += 1
+
+          return True
+
+      def handle_gallery(c, fd):
+          if c == '\x1b':
+              if select.select([fd], [], [], 0.05)[0]:
+                  c2 = os.read(fd, 1).decode('latin-1')
+                  if c2 == '[':
+                      c3 = os.read(fd, 1).decode('latin-1')
+                      if c3 == 'A':
+                          ui.gallery_sel = max(0, ui.gallery_sel - 1)
+                      elif c3 == 'B':
+                          ui.gallery_sel = min(max(0, len(ui.gallery) - 1), ui.gallery_sel + 1)
+              else:
+                  ui.mode = 'params'
+              return True
+
+          if c == 'q':
+              return False
+          elif c == 'b':
+              ui.mode = 'params'
+          elif c == 'j':
+              ui.gallery_sel = min(max(0, len(ui.gallery) - 1), ui.gallery_sel + 1)
+          elif c == 'k':
+              ui.gallery_sel = max(0, ui.gallery_sel - 1)
+          elif c == 'o' and ui.gallery:
+              import subprocess
+              path = ui.gallery[ui.gallery_sel]['path']
+              try:
+                  subprocess.Popen(['xdg-open', path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                  ui.status = "Opened: " + os.path.basename(path)
+              except:
+                  ui.status = "Could not open video"
+
+          # Keep gallery_scroll in range
+          _, height = get_size()
+          gallery_h = height - 6
+          if ui.gallery_sel < ui.gallery_scroll:
+              ui.gallery_scroll = ui.gallery_sel
+          elif ui.gallery_sel >= ui.gallery_scroll + gallery_h:
+              ui.gallery_scroll = ui.gallery_sel - gallery_h + 1
+
+          return True
+
+      # ========== One-shot mode ==========
+      if ui.prompt and context.get('prompt'):
+          # If prompt provided via CLI args, generate immediately
+          generate_videos()
+          context['output'] = ui.status
+          context['messages'] = context.get('messages', [])
+
+      # ========== Main TUI Loop ==========
+      elif not sys.stdin.isatty():
+          context['output'] = "Roll requires an interactive terminal."
+      else:
+          fd = sys.stdin.fileno()
+          old_settings = termios.tcgetattr(fd)
+
+          try:
+              tty.setcbreak(fd)
+              sys.stdout.write('\033[?25l')
+              sys.stdout.write('\033[2J')
+              render_screen()
+
+              running = True
+              while running:
+                  c = os.read(fd, 1).decode('latin-1')
+                  running = handle_input(c, fd)
+                  render_screen()
+
+          finally:
+              termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+              sys.stdout.write('\033[?25h')
+              sys.stdout.write('\033[2J\033[H')
+              sys.stdout.flush()
+
+          context['output'] = ui.status
+          context['messages'] = context.get('messages', [])