nimcode 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
nimcode/__init__.py ADDED
File without changes
nimcode/agent.py ADDED
@@ -0,0 +1,582 @@
1
+ from typing import List, Dict, Any, Optional
2
+ import os
3
+ import logging
4
+ from .nim_client import NimClient
5
+ from .lenient_parser import LenientParser
6
+ from .tools import ToolRegistry
7
+ from .permissions import PermissionEngine, PermissionMode
8
+ from .config import load_settings, save_global_setting
9
+ from .mcp_client import MCPManager
10
+ from .memory import MemoryManager
11
+
12
+ logger = logging.getLogger(__name__)
13
+
14
+ SYSTEM_PROMPT = """You are nimcode, an autonomous AI coding assistant.
15
+ You have access to a set of tools to read, write, and execute code.
16
+
17
+ CRITICAL INSTRUCTION FOR TOOL CALLING:
18
+ You must output exactly ONE tool call per turn, formatted exactly as a fenced XML block.
19
+ DO NOT use Markdown code blocks for the tool call. DO NOT output multiple tool calls in a single turn.
20
+
21
+ Format:
22
+ <tool_call>
23
+ {"tool": "ToolName", "args": {"arg1": "val1"}}
24
+ </tool_call>
25
+
26
+ Available Tools:
27
+ - Bash: {"tool": "Bash", "args": {"command": "string"}}
28
+ - Read: {"tool": "Read", "args": {"file_path": "string"}}
29
+ - Write: {"tool": "Write", "args": {"file_path": "string", "content": "string"}}
30
+ - Edit: {"tool": "Edit", "args": {"file_path": "string", "old_string": "string", "new_string": "string"}}
31
+ - Glob: {"tool": "Glob", "args": {"pattern": "string"}}
32
+ - Grep: {"tool": "Grep", "args": {"query": "string", "directory": "string"}}
33
+
34
+ Workspace Guidelines:
35
+ - When creating plans, store them inside the `.nimcode/plans/` directory (e.g., `.nimcode/plans/dino_game_plan.md`). Create this directory if it doesn't exist.
36
+ - When creating or learning new skills/memories, store them as markdown files inside the `.nimcode/skills/` directory.
37
+
38
+ When you have completely fulfilled the user's request and have no more tools to run, output the word TASK_COMPLETE.
39
+ """
40
+
41
+ class Agent:
42
+ def __init__(self, api_key: str, model: str = None, max_turns: int = 30, permission_mode: PermissionMode = PermissionMode.DEFAULT, max_tokens: int = 4000):
43
+ # Load global settings
44
+ self.settings = load_settings()
45
+ self.model = model or self.settings.get("model", "meta/llama-3.1-70b-instruct")
46
+ self.client = NimClient(api_key=api_key, model=self.model)
47
+
48
+ # Initialize MCP Manager
49
+ self.mcp = MCPManager(self.settings)
50
+
51
+ # Base system prompt
52
+ final_prompt = SYSTEM_PROMPT + self.mcp.get_system_prompt_additions()
53
+
54
+ # Load skills if present
55
+ skills_dir = os.path.join(os.getcwd(), ".nimcode", "skills")
56
+ if os.path.exists(skills_dir) and os.path.isdir(skills_dir):
57
+ loaded_skills = []
58
+ for filename in os.listdir(skills_dir):
59
+ if filename.endswith(".md"):
60
+ with open(os.path.join(skills_dir, filename), "r", encoding="utf-8") as f:
61
+ loaded_skills.append(f"--- SKILL: {filename} ---\n{f.read()}\n")
62
+ if loaded_skills:
63
+ final_prompt += "\n\nCRITICAL USER SKILLS & GUIDELINES:\n" + "\n".join(loaded_skills)
64
+ logger.info(f"Loaded {len(loaded_skills)} custom skills from {skills_dir}")
65
+
66
+ # Git context
67
+ if os.path.exists(".git"):
68
+ import subprocess
69
+ try:
70
+ branch = subprocess.check_output(["git", "branch", "--show-current"], text=True).strip()
71
+ status = subprocess.check_output(["git", "status", "-s"], text=True).strip()
72
+ final_prompt += f"\n\nGIT CONTEXT:\nBranch: {branch}\nUncommitted changes:\n{status if status else 'None'}"
73
+ except Exception as e:
74
+ logger.error(f"Failed to load git context: {e}")
75
+
76
+ self.messages: List[Dict[str, Any]] = [
77
+ {"role": "system", "content": final_prompt}
78
+ ]
79
+ self.max_turns = max_turns
80
+ self.permission_engine = PermissionEngine(mode=permission_mode)
81
+ self.memory = MemoryManager(max_tokens=max_tokens)
82
+
83
+ def save_history(self):
84
+ """Saves current conversation to NIMCODE.md"""
85
+ try:
86
+ import json
87
+ with open("NIMCODE.md", "w", encoding="utf-8") as f:
88
+ json.dump(self.messages, f)
89
+ except Exception as e:
90
+ logger.error(f"Failed to save history: {e}")
91
+
92
+ def load_history(self):
93
+ """Loads conversation from NIMCODE.md"""
94
+ try:
95
+ import json
96
+ if os.path.exists("NIMCODE.md"):
97
+ with open("NIMCODE.md", "r", encoding="utf-8") as f:
98
+ self.messages = json.load(f)
99
+ except Exception as e:
100
+ logger.error(f"Failed to load history: {e}")
101
+
102
+ async def _stream_response(self) -> str:
103
+ from rich.live import Live
104
+ from rich.markdown import Markdown
105
+ from rich.console import Console
106
+
107
+ c = Console()
108
+ response_text = ""
109
+ code_theme = self.settings.get("theme", "monokai")
110
+
111
+ first_chunk = None
112
+
113
+ with c.status("[bold cyan]🧠 NimCode is thinking...[/bold cyan]", spinner="dots"):
114
+ iterator = self.client.chat(self.messages).__aiter__()
115
+ try:
116
+ first_chunk = await iterator.__anext__()
117
+ except StopAsyncIteration:
118
+ pass
119
+ except KeyboardInterrupt:
120
+ c.print("\n[yellow]Generation interrupted before starting.[/yellow]")
121
+ return ""
122
+
123
+ if first_chunk is None:
124
+ return ""
125
+
126
+ response_text = first_chunk
127
+
128
+ with Live(Markdown(response_text, code_theme=code_theme), console=c, refresh_per_second=15) as live:
129
+ try:
130
+ async for chunk in iterator:
131
+ response_text += chunk
132
+ live.update(Markdown(response_text, code_theme=code_theme))
133
+ except KeyboardInterrupt:
134
+ live.update(Markdown(response_text + "\n\n*[yellow]Stream interrupted by user.[/yellow]*", code_theme=code_theme))
135
+ c.print("\n[yellow]Generation interrupted.[/yellow]")
136
+
137
+ # Approximate Token Tracker Update
138
+ est_tokens = len(response_text) // 4
139
+ if not hasattr(self, "session_tokens"):
140
+ self.session_tokens = 0
141
+ self.session_tokens += est_tokens
142
+
143
+ # Approximate cost based on 70B typical rates ($3/1M tokens)
144
+ cost = (self.session_tokens / 1000000) * 3.0
145
+ c.print(f"[dim]Output est. tokens: {est_tokens} | Session Cost: ~${cost:.4f}[/dim]")
146
+
147
+ # Context usage warning
148
+ import json
149
+ total_context_chars = sum(len(str(m.get("content", ""))) for m in self.messages)
150
+ total_est_tokens = total_context_chars // 4
151
+ max_context = 128000 # Assume standard Llama-3.1 128k context for now
152
+ if total_est_tokens > max_context * 0.8:
153
+ c.print("[bold yellow]⚠️ Context window is over 80% full. Consider running /compact or /clear.[/bold yellow]")
154
+
155
+ return response_text
156
+
157
+ async def run(self, initial_prompt: str = None) -> None:
158
+ """Main execution loop for NimCode agent."""
159
+ if hasattr(self.mcp, "connect_all"):
160
+ await self.mcp.connect_all()
161
+
162
+ if initial_prompt:
163
+ self.messages.append({"role": "user", "content": initial_prompt})
164
+
165
+ cwd = os.getcwd()
166
+ turn = 0
167
+ while turn < self.max_turns:
168
+ turn += 1
169
+ logger.info(f"--- Turn {turn} ---")
170
+
171
+ # Compact context before calling API
172
+ self.messages = self.memory.compact_context(self.messages)
173
+
174
+ # We don't pass tools to the native API for fallback mode.
175
+ # We expect the model to output <tool_call> block in the content.
176
+ full_content = ""
177
+ try:
178
+ full_content = await self._stream_response()
179
+ except Exception as e:
180
+ logger.error(f"Error calling NIM API: {e}")
181
+ break
182
+
183
+ self.messages.append({"role": "assistant", "content": full_content})
184
+
185
+ # Log turn to NIMCODE.md
186
+ try:
187
+ # We log the user's latest prompt, or the tool output
188
+ last_user_msg = self.messages[-2].get("content", "") if len(self.messages) > 1 else ""
189
+ MemoryManager.log_to_nimcode_md(turn, last_user_msg, full_content, cwd)
190
+ except Exception as e:
191
+ logger.error(f"Failed to log to NIMCODE.md: {e}")
192
+
193
+ if "TASK_COMPLETE" in full_content:
194
+ logger.info("Agent finished task.")
195
+ break
196
+
197
+ try:
198
+ prose, tool_calls = LenientParser.process_model_response(full_content)
199
+
200
+ if not tool_calls:
201
+ # Model responded with plain text but didn't say TASK_COMPLETE.
202
+ self.messages.append({"role": "user", "content": "Please continue. Use a tool or output TASK_COMPLETE."})
203
+ continue
204
+
205
+ # We execute the first tool call (ignoring others if it hallucinated multiple)
206
+ tool_call = tool_calls[0]
207
+ tool_name = tool_call.get("tool", "Unknown")
208
+
209
+ logger.info(f"Checking permissions for tool: {tool_name}")
210
+ if not self.permission_engine.check_permission(tool_call):
211
+ self.messages.append({
212
+ "role": "user",
213
+ "content": f"User explicitly denied permission to execute {tool_name}. Please choose another approach."
214
+ })
215
+ continue
216
+
217
+ logger.info(f"Running tool: {tool_name}")
218
+ from rich.console import Console
219
+ c = Console()
220
+ with c.status(f"[bold magenta]⚙️ Running {tool_name}...[/bold magenta]", spinner="bouncingBar"):
221
+ if not ToolRegistry.get_tool_schema(tool_name):
222
+ # Attempt MCP
223
+ try:
224
+ mcp_result = await self.mcp.call_tool_by_name(tool_name, tool_call.get("args", {}))
225
+ # mcp_result.content is a list of CallToolResult objects
226
+ result = "\n".join([str(c.text) for c in mcp_result.content if hasattr(c, 'text')])
227
+ if not result:
228
+ result = str(mcp_result)
229
+ except Exception as e:
230
+ result = f"Error executing MCP tool {tool_name}: {e}"
231
+ else:
232
+ result = ToolRegistry.execute(tool_call)
233
+
234
+ # Auto-Linting
235
+ if tool_name in ["Write", "Edit"] and "Error" not in result:
236
+ file_path = tool_call.get("args", {}).get("file_path", "")
237
+ if file_path.endswith(".py"):
238
+ import subprocess
239
+ subprocess.run(["black", "-q", file_path], capture_output=True)
240
+ elif file_path.endswith((".js", ".ts", ".jsx", ".tsx")):
241
+ import subprocess
242
+ subprocess.run(["npx", "prettier", "--write", file_path], capture_output=True)
243
+
244
+ # We simulate tool messages by adding a user message with the tool result.
245
+ # In native mode this would be a "tool" role message, but for fallback mode,
246
+ # passing it as a user message makes it explicit.
247
+ self.messages.append({
248
+ "role": "user",
249
+ "content": f"Tool {tool_name} returned:\n{result}"
250
+ })
251
+
252
+ except Exception as e:
253
+ logger.error(f"Error parsing/executing tool: {e}")
254
+ self.messages.append({
255
+ "role": "user",
256
+ "content": f"Your tool call was malformed or failed: {e}. Please fix the JSON syntax and try again."
257
+ })
258
+
259
+ if turn >= self.max_turns:
260
+ logger.warning(f"Max turns ({self.max_turns}) reached.")
261
+
262
+ self.save_history()
263
+
264
+ async def start_repl(self) -> None:
265
+ from prompt_toolkit import PromptSession
266
+ from prompt_toolkit.history import FileHistory
267
+ import os
268
+ from prompt_toolkit.styles import Style
269
+ from rich.console import Console
270
+ from rich.panel import Panel
271
+
272
+ # Connect MCPs before REPL
273
+ if hasattr(self.mcp, "connect_all"):
274
+ await self.mcp.connect_all()
275
+
276
+ history_path = os.path.join(os.getcwd(), ".nimcode", "history")
277
+ if not os.path.exists(os.path.dirname(history_path)):
278
+ os.makedirs(os.path.dirname(history_path), exist_ok=True)
279
+
280
+ session = PromptSession(history=FileHistory(history_path))
281
+
282
+ console = Console()
283
+
284
+ # Build Dashboard Splash
285
+ import subprocess
286
+ project_name = os.path.basename(os.getcwd())
287
+ branch = "Unknown"
288
+ if os.path.exists(".git"):
289
+ try:
290
+ branch = subprocess.check_output(["git", "branch", "--show-current"], text=True, stderr=subprocess.DEVNULL).strip()
291
+ except:
292
+ pass
293
+
294
+ splash_text = (
295
+ f"[bold cyan]Project:[/bold cyan] {project_name}\n"
296
+ f"[bold cyan]Branch:[/bold cyan] {branch}\n"
297
+ f"[bold cyan]Model:[/bold cyan] {self.model}\n\n"
298
+ f"Type [yellow]/help[/yellow] to see available commands."
299
+ )
300
+ console.print(Panel.fit(splash_text, title="[bold green]NimCode Agent[/bold green]", border_style="green"))
301
+
302
+ style = Style.from_dict({
303
+ 'prompt': '#00aa00 bold',
304
+ })
305
+ session = PromptSession(style=style)
306
+
307
+ while True:
308
+ try:
309
+ user_input = await session.prompt_async("NimCode> ")
310
+ if user_input.lower() in ["/exit", "/quit"]:
311
+ self.save_history()
312
+ break
313
+ elif user_input.strip() == "/help":
314
+ from rich.table import Table
315
+ table = Table(title="NimCode Commands", show_header=True, header_style="bold magenta")
316
+ table.add_column("Command", style="cyan", width=15)
317
+ table.add_column("Description", style="white")
318
+
319
+ table.add_row("/help", "Show this help menu")
320
+ table.add_row("/plan", "Enter planning mode (safe mode)")
321
+ table.add_row("/code", "Enter coding mode (all tools enabled)")
322
+ table.add_row("/models", "Select NVIDIA NIM model")
323
+ table.add_row("/theme <name>", "Change syntax theme (e.g., monokai)")
324
+ table.add_row("/clear", "Clear current context history")
325
+ table.add_row("/compact", "Compact context to save tokens")
326
+ table.add_row("/commit", "Auto-generate git commit message")
327
+ table.add_row("/fix <cmd>", "Run command and fix errors automatically")
328
+ table.add_row("/testgen <file>", "Generate 100% coverage tests for a file")
329
+ table.add_row("/vision", "Capture screen and analyze with Vision AI")
330
+ table.add_row("/voice", "Speak to NimCode (records for 5 seconds)")
331
+ table.add_row("/index", "Index project files for Semantic Search")
332
+ table.add_row("/exit", "Exit NimCode")
333
+
334
+ console.print(table)
335
+ continue
336
+ elif user_input.strip() == "/clear":
337
+ self.messages = [self.messages[0]]
338
+ console.print("[yellow]Context cleared.[/yellow]")
339
+ continue
340
+ elif user_input.strip() == "/compact":
341
+ self.messages = self.memory.compact_context(self.messages)
342
+ console.print("[yellow]Context compacted.[/yellow]")
343
+ continue
344
+ elif user_input.strip() == "/plan":
345
+ console.print("[bold blue]Entering Plan mode.[/bold blue] Mutating tools will be denied by default.")
346
+ self.permission_engine.mode = PermissionMode.DEFAULT
347
+ self.messages.append({"role": "system", "content": "You are now in planning mode. Use Read tools to explore. Then use the Write tool to write a markdown plan file inside the '.nimcode/plans/' directory (e.g., '.nimcode/plans/feature_x_plan.md'). Do NOT use Bash or Edit tools."})
348
+ continue
349
+ elif user_input.strip() == "/code":
350
+ console.print("[bold magenta]Entering Code mode.[/bold magenta] Standard permissions restored.")
351
+ self.messages.append({"role": "system", "content": "You are now in coding mode. You may use all available tools."})
352
+ continue
353
+ elif user_input.strip().startswith("/theme"):
354
+ parts = user_input.strip().split()
355
+ if len(parts) > 1:
356
+ self.settings["theme"] = parts[1]
357
+ save_global_setting("theme", parts[1])
358
+ console.print(f"[green]Theme updated to '{parts[1]}'[/green]")
359
+ else:
360
+ console.print("[yellow]Usage: /theme <name> (e.g., /theme monokai)[/yellow]")
361
+ continue
362
+ elif user_input.strip() == "/models":
363
+ console.print("[bold yellow]Fetching available models from NVIDIA NIM...[/bold yellow]")
364
+ models = await self.client.get_available_models()
365
+ for i, m in enumerate(models):
366
+ console.print(f"[{i+1}] {m}")
367
+
368
+ selection = await session.prompt_async("Select a model by number (or press enter to cancel): ")
369
+ if selection.strip().isdigit():
370
+ idx = int(selection.strip()) - 1
371
+ if 0 <= idx < len(models):
372
+ selected_model = models[idx]
373
+ self.client.model = selected_model
374
+ self.model = selected_model
375
+ save_global_setting("model", selected_model)
376
+ console.print(f"[bold green]Model changed to: {selected_model}[/bold green]")
377
+ else:
378
+ console.print("[red]Invalid selection.[/red]")
379
+ else:
380
+ console.print("[yellow]Model selection cancelled.[/yellow]")
381
+ continue
382
+ elif user_input.strip() == "/commit":
383
+ if not os.path.exists(".git"):
384
+ console.print("[red]Not a git repository.[/red]")
385
+ continue
386
+ console.print("[bold yellow]Generating commit message...[/bold yellow]")
387
+ import subprocess
388
+ diff = subprocess.check_output(["git", "diff", "--cached"], text=True)
389
+ if not diff:
390
+ diff = subprocess.check_output(["git", "diff"], text=True)
391
+ if not diff:
392
+ console.print("[yellow]No changes to commit.[/yellow]")
393
+ continue
394
+ prompt = f"Write a concise, professional git commit message for these changes:\n\n{diff[:3000]}\n\nOnly output the commit message string, nothing else."
395
+ msg = await self.client.chat_one_shot(prompt)
396
+ console.print(f"[bold green]Suggested Commit:[/bold green]\n{msg}")
397
+ console.print("\nTo commit, run: [bold cyan]git commit -m \"...\"[/bold cyan]")
398
+ continue
399
+ elif user_input.strip().startswith("/testgen"):
400
+ parts = user_input.strip().split(" ", 1)
401
+ if len(parts) > 1:
402
+ filepath = parts[1]
403
+ if os.path.exists(filepath):
404
+ console.print(f"[bold cyan]Generating tests for {filepath}...[/bold cyan]")
405
+ prompt = f"Read the file `{filepath}` using the Read tool if necessary. Then write a comprehensive suite of unit tests for it with 100% coverage. Write the tests to a new file (e.g. `test_{os.path.basename(filepath)}` for Python). Make sure they pass."
406
+ await self.run(prompt)
407
+ else:
408
+ console.print(f"[red]File {filepath} not found.[/red]")
409
+ else:
410
+ console.print("[yellow]Usage: /testgen <filepath>[/yellow]")
411
+ continue
412
+ elif user_input.strip().startswith("/fix"):
413
+ parts = user_input.strip().split(" ", 1)
414
+ if len(parts) < 2:
415
+ console.print("[yellow]Usage: /fix <command> (e.g. /fix pytest)[/yellow]")
416
+ continue
417
+
418
+ cmd = parts[1]
419
+ console.print(f"[bold cyan]Running {cmd}...[/bold cyan]")
420
+ import subprocess
421
+ for attempt in range(3):
422
+ result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
423
+ if result.returncode == 0:
424
+ console.print("[bold green]Command passed![/bold green]")
425
+ break
426
+ else:
427
+ console.print(f"[bold red]Command failed (attempt {attempt+1}/3). Fixing...[/bold red]")
428
+ await self.run(f"The command `{cmd}` failed with exit code {result.returncode}.\nStderr: {result.stderr[-2000:]}\nStdout: {result.stdout[-2000:]}\nFix the code so it passes.")
429
+ continue
430
+
431
+ # Check for aliases
432
+ aliases = self.settings.get("aliases", {})
433
+ first_word = user_input.strip().split()[0] if user_input.strip() else ""
434
+ if first_word in aliases:
435
+ user_input = user_input.replace(first_word, aliases[first_word], 1)
436
+ console.print(f"[dim]Aliased to: {user_input}[/dim]")
437
+
438
+ if user_input.strip().startswith("/config"):
439
+ parts = user_input.strip().split()
440
+ if len(parts) >= 3 and parts[1] == "set":
441
+ key = parts[2]
442
+ val = " ".join(parts[3:])
443
+ self.settings[key] = val
444
+ save_global_setting(key, val)
445
+ console.print(f"[bold green]✓ Setting '{key}' updated to '{val}'[/bold green]")
446
+ else:
447
+ console.print("[yellow]Usage: /config set <key> <value>[/yellow]")
448
+ continue
449
+ elif user_input.strip().startswith("/alias"):
450
+ parts = user_input.strip().split("=", 1)
451
+ if len(parts) == 2:
452
+ name = parts[0].replace("/alias", "").strip()
453
+ cmd = parts[1].strip()
454
+ aliases = self.settings.get("aliases", {})
455
+ aliases[name] = cmd
456
+ self.settings["aliases"] = aliases
457
+ save_global_setting("aliases", aliases)
458
+ console.print(f"[bold green]✓ Alias '{name}' set to '{cmd}'[/bold green]")
459
+ else:
460
+ console.print("[yellow]Usage: /alias <name> = <command>[/yellow]")
461
+ continue
462
+ elif user_input.strip().startswith("/add"):
463
+ parts = user_input.strip().split(" ", 1)
464
+ if len(parts) > 1:
465
+ filepath = parts[1]
466
+ try:
467
+ with open(filepath, "r", encoding="utf-8") as f:
468
+ content = f.read()
469
+ self.messages.append({"role": "system", "content": f"Pinned File ({filepath}):\n\n{content}"})
470
+ console.print(f"[bold green]✓ {filepath} pinned to context.[/bold green]")
471
+ except Exception as e:
472
+ console.print(f"[bold red]Failed to read {filepath}: {e}[/bold red]")
473
+ else:
474
+ console.print("[yellow]Usage: /add <filepath>[/yellow]")
475
+ continue
476
+ elif user_input.strip().startswith("/rewind"):
477
+ parts = user_input.strip().split(" ", 1)
478
+ turns = 1
479
+ if len(parts) > 1 and parts[1].isdigit():
480
+ turns = int(parts[1])
481
+
482
+ items_to_remove = turns * 2
483
+ if len(self.messages) > items_to_remove:
484
+ self.messages = self.messages[:-items_to_remove]
485
+ console.print(f"[bold green]⏪ Rewound {turns} turns.[/bold green]")
486
+ else:
487
+ self.messages = [self.messages[0]]
488
+ console.print("[bold green]⏪ Rewound to beginning.[/bold green]")
489
+ continue
490
+ elif user_input.strip() == "/fork":
491
+ import subprocess
492
+ import time
493
+ branch_name = f"nimcode-fork-{int(time.time())}"
494
+ try:
495
+ subprocess.run(["git", "checkout", "-b", branch_name], check=True, capture_output=True)
496
+ console.print(f"[bold green]🔀 Forked conversation to new git branch: {branch_name}[/bold green]")
497
+ except subprocess.CalledProcessError:
498
+ console.print("[bold red]Failed to create branch. Are you in a git repo with commits?[/bold red]")
499
+ continue
500
+ elif user_input.strip() == "/vision":
501
+ console.print("[bold cyan]👁️ Capturing screen...[/bold cyan]")
502
+ import pyautogui
503
+ import io
504
+ import base64
505
+ try:
506
+ screenshot = pyautogui.screenshot()
507
+ buffered = io.BytesIO()
508
+ screenshot.thumbnail((1920, 1080))
509
+ screenshot.save(buffered, format="JPEG", quality=80)
510
+ img_str = base64.b64encode(buffered.getvalue()).decode()
511
+
512
+ console.print("[bold cyan]🧠 Analyzing image with Vision Model...[/bold cyan]")
513
+ vision_response = await self.client.chat_vision(img_str, "Describe what is on the screen in detail. If there is code, explain what it does or if there are any visible errors.")
514
+ console.print(f"\n[bold magenta]Vision Analysis:[/bold magenta]\n{vision_response}\n")
515
+ self.messages.append({"role": "system", "content": f"The user provided a screenshot. The vision model analyzed it as: {vision_response}"})
516
+ except Exception as e:
517
+ console.print(f"[bold red]Vision failed: {e}[/bold red]")
518
+ continue
519
+ elif user_input.strip() == "/voice":
520
+ console.print("[bold cyan]🎤 Recording for 5 seconds... Speak now![/bold cyan]")
521
+ import sounddevice as sd
522
+ import soundfile as sf
523
+ import os
524
+ import speech_recognition as sr
525
+ try:
526
+ fs = 16000
527
+ seconds = 5
528
+ myrecording = sd.rec(int(seconds * fs), samplerate=fs, channels=1)
529
+ sd.wait()
530
+ console.print("[bold cyan]⏳ Processing audio...[/bold cyan]")
531
+ sf.write('temp_voice.wav', myrecording, fs)
532
+
533
+ r = sr.Recognizer()
534
+ with sr.AudioFile('temp_voice.wav') as source:
535
+ audio = r.record(source)
536
+ try:
537
+ transcription = r.recognize_google(audio)
538
+ console.print(f"[bold green]🗣️ You said:[/bold green] {transcription}")
539
+ await self.run(transcription)
540
+ except sr.UnknownValueError:
541
+ console.print("[bold red]Could not understand audio.[/bold red]")
542
+ except Exception as e:
543
+ console.print(f"[bold red]Voice failed: {e}[/bold red]")
544
+ continue
545
+ elif user_input.strip() == "/index":
546
+ console.print("[bold cyan]🔍 Indexing project files for Semantic Search...[/bold cyan]")
547
+ self.search_index = {}
548
+ import glob
549
+ import os
550
+ for ext in ["*.py", "*.js", "*.ts", "*.md"]:
551
+ for file in glob.glob(f"**/{ext}", recursive=True):
552
+ if "node_modules" in file or "venv" in file or ".git" in file:
553
+ continue
554
+ try:
555
+ with open(file, "r", encoding="utf-8") as f:
556
+ self.search_index[file] = f.read()
557
+ except:
558
+ pass
559
+ console.print(f"[bold green]✓ Indexed {len(self.search_index)} files.[/bold green]")
560
+ # Expose the index as a string summary for the agent to know what's there
561
+ index_summary = ", ".join(self.search_index.keys())
562
+ self.messages.append({"role": "system", "content": f"Project is indexed. Available indexed files: {index_summary}"})
563
+ continue
564
+ elif user_input.strip().startswith("/"):
565
+ import difflib
566
+ valid_commands = ["/help", "/plan", "/code", "/models", "/theme", "/clear", "/compact", "/commit", "/fix", "/exit", "/quit", "/config", "/alias", "/add", "/rewind", "/fork", "/testgen", "/vision", "/voice", "/index"]
567
+ cmd_name = user_input.strip().split()[0]
568
+ matches = difflib.get_close_matches(cmd_name, valid_commands, n=1, cutoff=0.5)
569
+ if matches:
570
+ console.print(f"[yellow]Unknown command '{cmd_name}'. Did you mean [bold cyan]{matches[0]}[/bold cyan]?[/yellow]")
571
+ else:
572
+ console.print(f"[red]Unknown command '{cmd_name}'. Type /help for a list of commands.[/red]")
573
+ continue
574
+
575
+ if not user_input.strip():
576
+ continue
577
+
578
+ await self.run(user_input)
579
+
580
+ except (EOFError, KeyboardInterrupt):
581
+ break
582
+