npcpy 1.2.36__py3-none-any.whl → 1.3.1__py3-none-any.whl

npcpy/npc_compiler.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from pyexpat.errors import messages
 import yaml
 import json
@@ -18,7 +19,6 @@ from typing import Any, Dict, List, Optional, Union, Callable, Tuple
 from jinja2 import Environment, FileSystemLoader, Template, Undefined, DictLoader
 from sqlalchemy import create_engine, text
 import npcpy as npy 
-from npcpy.llm_funcs import DEFAULT_ACTION_SPACE
 from npcpy.tools import auto_tools
 import math 
 import random
@@ -183,6 +183,7 @@ def initialize_npc_project(
     """Initialize an NPC project"""
     if directory is None:
         directory = os.getcwd()
+    directory = os.path.expanduser(os.fspath(directory))
 
     npc_team_dir = os.path.join(directory, "npc_team")
     os.makedirs(npc_team_dir, exist_ok=True)
@@ -191,7 +192,8 @@ def initialize_npc_project(
                    "assembly_lines", 
                    "sql_models", 
                    "jobs", 
-                   "triggers"]:
+                   "triggers",
+                   "tools"]:
         os.makedirs(os.path.join(npc_team_dir, subdir), exist_ok=True)
     
     forenpc_path = os.path.join(npc_team_dir, "forenpc.npc")
@@ -206,20 +208,166 @@ def initialize_npc_project(
         }
         with open(forenpc_path, "w") as f:
             yaml.dump(default_npc, f)
-    ctx_path = os.path.join(npc_team_dir, "team.ctx")
-    if not os.path.exists(ctx_path):
+    parsed_templates: List[str] = []
+    if templates:
+        if isinstance(templates, str):
+            parsed_templates = [
+                t.strip() for t in re.split(r"[,\s]+", templates) if t.strip()
+            ]
+        elif isinstance(templates, (list, tuple, set)):
+            parsed_templates = [str(t).strip() for t in templates if str(t).strip()]
+        else:
+            parsed_templates = [str(templates).strip()]
+
+    ctx_destination: Optional[str] = None
+    preexisting_ctx = [
+        os.path.join(npc_team_dir, f)
+        for f in os.listdir(npc_team_dir)
+        if f.endswith(".ctx")
+    ]
+    if preexisting_ctx:
+        ctx_destination = preexisting_ctx[0]
+        if len(preexisting_ctx) > 1:
+            print(
+                "Warning: Multiple .ctx files already present; using first and ignoring the rest."
+            )
+    
+    def _resolve_template_path(template_name: str) -> Optional[str]:
+        expanded = os.path.expanduser(template_name)
+        if os.path.exists(expanded):
+            return expanded
+
+        embedded_templates = {
+            "slean": """name: slean
+primary_directive: You are slean, the marketing innovator AI. Your responsibility is to create marketing campaigns and manage them effectively, while also thinking creatively to solve marketing challenges. Guide the strategy that drives customer engagement and brand awareness.
+""",
+            "turnic": """name: turnic
+primary_directive: Assist with sales challenges and questions. Opt for straightforward solutions that help sales professionals achieve quick results.
+""",
+            "budgeto": """name: budgeto
+primary_directive: You manage marketing budgets, ensuring resources are allocated efficiently and spend is optimized.
+""",
+            "relatio": """name: relatio
+primary_directive: You manage customer relationships and ensure satisfaction throughout the sales process. Focus on nurturing clients and maintaining long-term connections.
+""",
+            "funnel": """name: funnel
+primary_directive: You oversee the sales pipeline, track progress, and optimize conversion rates to move leads efficiently.
+""",
+        }
+
+        base_dirs = [
+            os.path.expanduser("~/.npcsh/npc_team/templates"),
+            os.path.expanduser("~/.npcpy/npc_team/templates"),
+            os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "tests", "template_tests", "npc_team")),
+            os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "examples", "npc_team")),
+            os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "example_npc_project", "npc_team")),
+        ]
+        base_dirs = [d for d in base_dirs if os.path.isdir(d)]
+
+        for base in base_dirs:
+            direct = os.path.join(base, template_name)
+            if os.path.exists(direct):
+                return direct
+            if not direct.endswith(".npc") and os.path.exists(direct + ".npc"):
+                return direct + ".npc"
+            for root, _, files in os.walk(base):
+                for fname in files:
+                    stem, ext = os.path.splitext(fname)
+                    if ext == ".npc" and stem == template_name:
+                        return os.path.join(root, fname)
+
+        # If no on-disk template found, fall back to embedded definitions
+        if template_name in embedded_templates:
+            embedded_dir = os.path.join(npc_team_dir, "_embedded_templates", template_name)
+            os.makedirs(embedded_dir, exist_ok=True)
+            npc_file = os.path.join(embedded_dir, f"{template_name}.npc")
+            if not os.path.exists(npc_file):
+                with open(npc_file, "w") as f:
+                    f.write(embedded_templates[template_name])
+            return embedded_dir
+        return None
+
+    def _copy_template(src_path: str) -> List[str]:
+        nonlocal ctx_destination
+        copied: List[str] = []
+        src_path = os.path.expanduser(src_path)
+
+        allowed_exts = {".npc", ".tool", ".pipe", ".sql", ".job", ".ctx", ".yaml", ".yml"}
+
+        if os.path.isfile(src_path):
+            if os.path.splitext(src_path)[1] in allowed_exts:
+                if os.path.splitext(src_path)[1] == ".ctx":
+                    if ctx_destination:
+                        print(
+                            f"Warning: Skipping extra context file '{src_path}' because one already exists."
+                        )
+                        return copied
+                    dest_path = os.path.join(npc_team_dir, os.path.basename(src_path))
+                    ctx_destination = dest_path
+                else:
+                    dest_path = os.path.join(npc_team_dir, os.path.basename(src_path))
+                if not os.path.exists(dest_path):
+                    shutil.copy2(src_path, dest_path)
+                copied.append(dest_path)
+            return copied
+
+        for root, _, files in os.walk(src_path):
+            rel_dir = os.path.relpath(root, src_path)
+            dest_dir = npc_team_dir if rel_dir == "." else os.path.join(npc_team_dir, rel_dir)
+            os.makedirs(dest_dir, exist_ok=True)
+            for fname in files:
+                if os.path.splitext(fname)[1] not in allowed_exts:
+                    continue
+                if os.path.splitext(fname)[1] == ".ctx":
+                    if ctx_destination:
+                        print(
+                            f"Warning: Skipping extra context file '{os.path.join(root, fname)}' because one already exists."
+                        )
+                        continue
+                    dest_path = os.path.join(npc_team_dir, fname)
+                    ctx_destination = dest_path
+                else:
+                    dest_path = os.path.join(dest_dir, fname)
+                if not os.path.exists(dest_path):
+                    shutil.copy2(os.path.join(root, fname), dest_path)
+                copied.append(dest_path)
+        return copied
+
+    applied_templates: List[str] = []
+    if parsed_templates:
+        for template_name in parsed_templates:
+            template_path = _resolve_template_path(template_name)
+            if not template_path:
+                print(f"Warning: Template '{template_name}' not found in known template directories.")
+                continue
+            copied = _copy_template(template_path)
+            if copied:
+                applied_templates.append(template_name)
+    
+    if applied_templates:
+        applied_templates = sorted(set(applied_templates))
+    if not ctx_destination:
+        default_ctx_path = os.path.join(npc_team_dir, "team.ctx")
         default_ctx = {
             'name': '',
-            'context' : '', 
+            'context' : context or '', 
             'preferences': '', 
             'mcp_servers': '', 
             'databases':'', 
             'use_global_jinxs': True,
             'forenpc': 'forenpc'
         }
-        with open(ctx_path, "w") as f:
+        if parsed_templates:
+            default_ctx['templates'] = parsed_templates
+        with open(default_ctx_path, "w") as f:
             yaml.dump(default_ctx, f)
-            
+        ctx_destination = default_ctx_path
+
+    if applied_templates:
+        return (
+            f"NPC project initialized in {npc_team_dir} "
+            f"using templates: {', '.join(applied_templates)}"
+        )
     return f"NPC project initialized in {npc_team_dir}"
 
 
@@ -236,7 +384,6 @@ def write_yaml_file(file_path, data):
         print(f"Error writing YAML file {file_path}: {e}")
         return False
 
-
 class Jinx:
     ''' 
     Jinx represents a workflow template with Jinja-rendered steps.
@@ -246,7 +393,8 @@ class Jinx:
     - inputs: list of input parameters
     - description: what the jinx does
     - npc: optional NPC to execute with
-    - steps: list of step definitions with code
+    - steps: list of step definitions with code. This section can now be a Jinja template itself.
+    - file_context: optional list of file patterns to include as context
     
     Execution:
     - Renders Jinja templates in step code with input values
@@ -261,9 +409,13 @@ class Jinx:
         else:
             raise ValueError("Either jinx_data or jinx_path must be provided")
         
-        # Keep a copy for macro expansion, but retain the executable steps by default
+        # _raw_steps will now hold the original, potentially templated, steps definition
         self._raw_steps = list(self.steps)
-        self.steps = list(self._raw_steps)
+        self.steps = [] # Will be populated after first-pass rendering
+        self.parsed_files = {}
+        if self.file_context:
+            self.parsed_files = self._parse_file_patterns(self.file_context)
+
     def _load_from_file(self, path):
         jinx_data = load_yaml_file(path)
         if not jinx_data:
@@ -283,9 +435,38 @@ class Jinx:
         self.inputs = jinx_data.get("inputs", [])
         self.description = jinx_data.get("description", "")
         self.npc = jinx_data.get("npc")
-        self.steps = jinx_data.get("steps", [])
+        self.steps = jinx_data.get("steps", []) # This can now be a Jinja templated list
+        self.file_context = jinx_data.get("file_context", [])
         self._source_path = jinx_data.get("_source_path", None)
 
+    def to_tool_def(self) -> Dict[str, Any]:
+        """Convert this Jinx to an OpenAI-style tool definition."""
+        properties = {}
+        required = []
+        for inp in self.inputs:
+            if isinstance(inp, str):
+                properties[inp] = {"type": "string", "description": f"Parameter: {inp}"}
+                required.append(inp)
+            elif isinstance(inp, dict):
+                name = list(inp.keys())[0]
+                default_val = inp.get(name, "")
+                desc = f"Parameter: {name}"
+                if default_val != "":
+                    desc += f" (default: {default_val})"
+                properties[name] = {"type": "string", "description": desc}
+        return {
+            "type": "function",
+            "function": {
+                "name": self.jinx_name,
+                "description": self.description or f"Jinx: {self.jinx_name}",
+                "parameters": {
+                    "type": "object",
+                    "properties": properties,
+                    "required": required
+                }
+            }
+        }
+
     def render_first_pass(
         self, 
         jinja_env_for_macros: Environment, 
@@ -293,21 +474,52 @@ class Jinx:
     ):
         """
         Performs the first-pass Jinja rendering on the Jinx's raw steps.
-        This expands nested Jinx calls (e.g., {{ sh(...) }} or 
-        engine: jinx_name) but preserves runtime variables 
-        (e.g., {{ command_var }}).
-        
-        Args:
-            jinja_env_for_macros: The Jinja Environment configured with 
-                Jinx callables in its globals.
-            all_jinx_callables: A dictionary of Jinx names to their 
-                callable functions (from create_jinx_callable).
+        This expands Jinja control flow (for, if) to generate step structures,
+        then expands nested Jinx calls (e.g., {{ sh(...) }} or engine: jinx_name)
+        and inline macros.
         """
-        rendered_steps_output = []
+        # 1. Join the list of raw steps (which are individual YAML lines) into a single string.
+        #    This single string is the complete Jinja template for the 'steps' section.
+        raw_steps_template_string = "\n".join(self._raw_steps)
+        
+        # 2. Render this single string as a Jinja template.
+        #    Jinja will now process the {% for %} and {% if %} directives,
+        #    dynamically generating the YAML structure.
+        try:
+            steps_template = jinja_env_for_macros.from_string(raw_steps_template_string)
+            # Pass globals (like num_tasks, include_greeting from Jinx inputs) 
+            # to the Jinja rendering context for structural templating.
+            rendered_steps_yaml_string = steps_template.render(**jinja_env_for_macros.globals)
+        except Exception as e:
+            # In a real Jinx, this would go to a proper logger.
+            # For this context, we handle the error gracefully.
+            # self._log_debug(f"Warning: Error during first-pass templating of Jinx '{self.jinx_name}' steps YAML: {e}")
+            self.steps = list(self._raw_steps) # Fallback to original raw steps
+            return
+
+        # 3. Parse the rendered YAML string back into a list of step dictionaries.
+        #    This step will now correctly interpret the YAML structure generated by Jinja.
+        try:
+            structurally_expanded_steps = yaml.safe_load(rendered_steps_yaml_string)
+            if not isinstance(structurally_expanded_steps, list):
+                # Handle cases where the rendered YAML might be empty or not a list
+                if structurally_expanded_steps is None:
+                    structurally_expanded_steps = []
+                else:
+                    raise ValueError(f"Rendered steps YAML did not result in a list: {type(structurally_expanded_steps)}")
+            self.steps = structurally_expanded_steps
+        except Exception as e:
+            # self._log_debug(f"Warning: Error re-parsing structurally expanded steps YAML for Jinx '{self.jinx_name}': {e}")
+            self.steps = list(self._raw_steps) # Fallback
+            return
 
-        for raw_step in self._raw_steps:
+        # 4. Now, iterate through these `structurally_expanded_steps` to expand
+        # declarative Jinx calls (engine: jinx_name) and inline macros.
+        # This is the second phase of the first-pass rendering.
+        final_rendered_steps = []
+        for raw_step in structurally_expanded_steps:
             if not isinstance(raw_step, dict):
-                rendered_steps_output.append(raw_step)
+                final_rendered_steps.append(raw_step)
                 continue
 
             engine_name = raw_step.get('engine')
@@ -326,51 +538,58 @@ class Jinx:
                     expanded_steps = yaml.safe_load(expanded_yaml_string)
                     
                     if isinstance(expanded_steps, list):
-                        rendered_steps_output.extend(expanded_steps)
-                        
+                        final_rendered_steps.extend(expanded_steps)
                     elif expanded_steps is not None:
-                        rendered_steps_output.append(expanded_steps)
-                        
+                        final_rendered_steps.append(expanded_steps)
                 except Exception as e:
-                    print(
-                        f"Warning: Error expanding Jinx '{engine_name}' "
-                        f"within Jinx '{self.jinx_name}' "
-                        f"(declarative): {e}"
-                    )
-                    rendered_steps_output.append(raw_step)
-            # Skip rendering for python/bash engine steps - preserve runtime variables
+                    # self._log_debug(
+                    #     f"Warning: Error expanding Jinx '{engine_name}' "
+                    #     f"within Jinx '{self.jinx_name}' "
+                    #     f"(declarative): {e}"
+                    # )
+                    final_rendered_steps.append(raw_step)
+            # For python/bash engine steps, only inline macro expansion happens in the next block.
+            # The code content itself is preserved for runtime Jinja rendering.
             elif raw_step.get('engine') in ['python', 'bash']:
-                rendered_steps_output.append(raw_step)
+                processed_step = {}
+                for key, value in raw_step.items():
+                    if isinstance(value, str):
+                        try:
+                            template = jinja_env_for_macros.from_string(value)
+                            # Render with empty context for inline macros/static values
+                            rendered_value = template.render({})
+                            try:
+                                loaded_value = yaml.safe_load(rendered_value)
+                                processed_step[key] = loaded_value
+                            except yaml.YAMLError:
+                                processed_step[key] = rendered_value
+                        except Exception as e:
+                            # self._log_debug(f"Warning: Error during first-pass rendering of Jinx '{self.jinx_name}' step field '{key}' (inline macro): {e}")
+                            processed_step[key] = value
+                    else:
+                        processed_step[key] = value
+                final_rendered_steps.append(processed_step)
             else:
-                # For other steps, do first-pass rendering (inline macro expansion)
+                # For other steps (e.g., custom engines, or just data), perform inline macro expansion
                 processed_step = {}
                 for key, value in raw_step.items():
                     if isinstance(value, str):
                         try:
-                            template = jinja_env_for_macros.from_string(
-                                value
-                            )
+                            template = jinja_env_for_macros.from_string(value)
                             rendered_value = template.render({})
-                            
                             try:
-                                loaded_value = yaml.safe_load(
-                                    rendered_value
-                                )
+                                loaded_value = yaml.safe_load(rendered_value)
                                 processed_step[key] = loaded_value
                             except yaml.YAMLError:
                                 processed_step[key] = rendered_value
                         except Exception as e:
-                            print(
-                                f"Warning: Error during first-pass "
-                                f"rendering of Jinx '{self.jinx_name}' "
-                                f"step field '{key}' (inline macro): {e}"
-                            )
+                            # self._log_debug(f"Warning: Error during first-pass rendering of Jinx '{self.jinx_name}' step field '{key}' (inline macro): {e}")
                             processed_step[key] = value
                     else:
                         processed_step[key] = value
-                rendered_steps_output.append(processed_step)
+                final_rendered_steps.append(processed_step)
         
-        self.steps = rendered_steps_output
+        self.steps = final_rendered_steps
 
     def execute(self,
                 input_values: Dict[str, Any],
@@ -399,6 +618,11 @@ class Jinx:
             "messages": messages,
             "npc": active_npc
         })
+        
+        # Add parsed file content to the context
+        if self.parsed_files:
+            context['file_context'] = self._format_parsed_files_context(self.parsed_files)
+            context['files'] = self.parsed_files # Also make raw dict available
 
         for i, step in enumerate(self.steps):
             context = self._execute_step(
@@ -409,9 +633,19 @@ class Jinx:
                 messages=messages,
                 extra_globals=extra_globals
             )
+            # If an error occurred in a step, propagate it and stop execution
+            if "error" in context.get("output", ""):
+                self._log_debug(f"DEBUG: Jinx '{self.jinx_name}' execution stopped due to error in step '{step.get('name', 'unnamed_step')}': {context['output']}")
+                break
 
         return context
 
+    def _log_debug(self, msg: str):
+        """Helper for logging debug messages to a file."""
+        log_file_path = os.path.expanduser("~/jinx_debug_log.txt")
+        with open(log_file_path, "a") as f:
+            f.write(f"[{datetime.now().isoformat()}] {msg}\n")
+
     def _execute_step(self,
                   step: Dict[str, Any],
                   context: Dict[str, Any],
@@ -420,40 +654,31 @@ class Jinx:
                   messages: Optional[List[Dict[str, str]]] = None,
                   extra_globals: Optional[Dict[str, Any]] = None):
         
-        def _log_debug(msg):
-            log_file_path = os.path.expanduser("~/jinx_debug_log.txt")
-            with open(log_file_path, "a") as f:
-                f.write(f"[{datetime.now().isoformat()}] {msg}\n")
-
         code_content = step.get("code", "")
         step_name = step.get("name", "unnamed_step")
         step_npc = step.get("npc")
         
         active_npc = step_npc if step_npc else npc
         
+        # Second pass Jinja rendering: render the step's code with the current runtime context
         try:
             template = jinja_env.from_string(code_content)
             rendered_code = template.render(**context)
         except Exception as e:
-            _log_debug(
-                f"Error rendering template for step {step_name} "
-                f"(second pass): {e}"
+            error_msg = (
+                f"Error rendering template for step '{step_name}' "
+                f"(second pass): {type(e).__name__}: {e}"
             )
-            rendered_code = code_content
+            context['output'] = error_msg
+            self._log_debug(error_msg)
+            return context
         
-        _log_debug(f"rendered jinx code: {rendered_code}")
-        _log_debug(
-            f"DEBUG: Before exec - rendered_code: {rendered_code}"
-        )
-        _log_debug(
-            f"DEBUG: Before exec - context['output'] before step: "
-            f"{context.get('output')}"
-        )
+        self._log_debug(f"DEBUG: Executing step '{step_name}' with rendered code: {rendered_code}")
 
         exec_globals = {
             "__builtins__": __builtins__,
             "npc": active_npc,
-            "context": context,
+            "context": context, # Pass context by reference
             "math": math, 
             "random": random, 
             "datetime": datetime,
@@ -477,43 +702,27 @@ class Jinx:
         if extra_globals:
             exec_globals.update(extra_globals)
         
-        exec_locals = {}
-        
+        exec_locals = {} # Locals for this specific exec call
+
         try:
             exec(rendered_code, exec_globals, exec_locals)
         except Exception as e:
             error_msg = (
-                f"Error executing step {step_name}: "
+                f"Error executing step '{step_name}': "
                 f"{type(e).__name__}: {e}"
             )
             context['output'] = error_msg
-            _log_debug(error_msg)
+            self._log_debug(error_msg)
             return context
         
-        _log_debug(f"DEBUG: After exec - exec_locals: {exec_locals}")
-        _log_debug(
-            f"DEBUG: After exec - 'output' in exec_locals: "
-            f"{'output' in exec_locals}"
-        )
-
+        # Update the main context with any variables set in exec_locals
         context.update(exec_locals)
         
-        _log_debug(
-            f"DEBUG: After context.update(exec_locals) - "
-            f"context['output']: {context.get('output')}"
-        )
-        _log_debug(f"context after jinx ex: {context}")
-
         if "output" in exec_locals:
             outp = exec_locals["output"]
             context["output"] = outp
             context[step_name] = outp
 
-            _log_debug(
-                f"DEBUG: Inside 'output' in exec_locals block - "
-                f"context['output']: {context.get('output')}"
-            )
-
             if messages is not None:
                 messages.append({
                     'role':'assistant', 
@@ -526,13 +735,95 @@ class Jinx:
         
         return context
         
+    def _parse_file_patterns(self, patterns_config):
+        """Parse file patterns configuration and load matching files into KV cache"""
+        if not patterns_config:
+            return {}
+        
+        file_cache = {}
+        
+        for pattern_entry in patterns_config:
+            if isinstance(pattern_entry, str):
+                pattern_entry = {"pattern": pattern_entry}
+            
+            pattern = pattern_entry.get("pattern", "")
+            recursive = pattern_entry.get("recursive", False)
+            base_path = pattern_entry.get("base_path", ".")
+            
+            if not pattern:
+                continue
+                
+            # Resolve base_path relative to jinx's source path or current working directory
+            if self._source_path:
+                base_path = os.path.join(os.path.dirname(self._source_path), base_path)
+            base_path = os.path.expanduser(base_path)
+            
+            if not os.path.isabs(base_path):
+                base_path = os.path.join(os.getcwd(), base_path)
+            
+            matching_files = self._find_matching_files(pattern, base_path, recursive)
+            
+            for file_path in matching_files:
+                file_content = self._load_file_content(file_path)
+                if file_content:
+                    relative_path = os.path.relpath(file_path, base_path)
+                    file_cache[relative_path] = file_content
+        
+        return file_cache
+
+    def _find_matching_files(self, pattern, base_path, recursive=False):
+        """Find files matching the given pattern"""
+        matching_files = []
+        
+        if not os.path.exists(base_path):
+            return matching_files
+        
+        if recursive:
+            for root, dirs, files in os.walk(base_path):
+                for filename in files:
+                    if fnmatch.fnmatch(filename, pattern):
+                        matching_files.append(os.path.join(root, filename))
+        else:
+            try:
+                for item in os.listdir(base_path):
+                    item_path = os.path.join(base_path, item)
+                    if os.path.isfile(item_path) and fnmatch.fnmatch(item, pattern):
+                        matching_files.append(item_path)
+            except PermissionError:
+                print(f"Permission denied accessing {base_path}")
+        
+        return matching_files
+
+    def _load_file_content(self, file_path):
+        """Load content from a file with error handling"""
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            print(f"Error reading {file_path}: {e}")
+            return None
+
+    def _format_parsed_files_context(self, parsed_files):
+        """Format parsed files into context string"""
+        if not parsed_files:
+            return ""
+        
+        context_parts = ["Additional context from files:"]
+        
+        for file_path, content in parsed_files.items():
+            context_parts.append(f"\n--- {file_path} ---")
+            context_parts.append(content)
+            context_parts.append("")
+        
+        return "\n".join(context_parts)
 
     def to_dict(self):
         result = {
             "jinx_name": self.jinx_name,
             "description": self.description,
             "inputs": self.inputs,
-            "steps": self._raw_steps 
+            "steps": self._raw_steps, # Save the original raw steps, which might be templated
+            "file_context": self.file_context
         }
         
         if self.npc:
@@ -578,6 +869,7 @@ class Jinx:
                 "jinx_name": name,
                 "description": doc.strip(),
                 "inputs": inputs,
+                "file_context": [],
                 "steps": [
                     {
                         "name": "mcp_function_call",
@@ -599,6 +891,8 @@ output = {mcp_tool.__module__}.{name}(
         except: 
             pass
 
+
+
 def load_jinxs_from_directory(directory):
     """Load all jinxs from a directory recursively"""
     jinxs = []
@@ -621,100 +915,12 @@ def load_jinxs_from_directory(directory):
 
 def jinx_to_tool_def(jinx_obj: 'Jinx') -> Dict[str, Any]:
     """Convert a Jinx instance into an MCP/LLM-compatible tool schema definition."""
-    properties: Dict[str, Any] = {}
-    required: List[str] = []
-    for inp in jinx_obj.inputs:
-        if isinstance(inp, str):
-            properties[inp] = {"type": "string"}
-            required.append(inp)
-        elif isinstance(inp, dict):
-            name = list(inp.keys())[0]
-            properties[name] = {"type": "string", "default": inp.get(name, "")}
-            required.append(name)
-    return {
-        "type": "function",
-        "function": {
-            "name": jinx_obj.jinx_name,
-            "description": jinx_obj.description or f"Jinx: {jinx_obj.jinx_name}",
-            "parameters": {
-                "type": "object",
-                "properties": properties,
-                "required": required
-            }
-        }
-    }
+    return jinx_obj.to_tool_def()
 
 def build_jinx_tool_catalog(jinxs: Dict[str, 'Jinx']) -> Dict[str, Dict[str, Any]]:
     """Helper to build a name->tool_def catalog from a dict of Jinx objects."""
     return {name: jinx_to_tool_def(jinx_obj) for name, jinx_obj in jinxs.items()}
 
-def get_npc_action_space(npc=None, team=None):
-    """Get action space for NPC including memory CRUD and core capabilities"""
-    actions = DEFAULT_ACTION_SPACE.copy()
-    
-    if npc:
-        core_tools = [
-            npc.think_step_by_step,
-        ]
-        if hasattr(npc, "write_code"):
-            core_tools.append(npc.write_code)
-        
-        if npc.command_history:
-            core_tools.extend([
-                npc.search_my_conversations,
-                npc.search_my_memories,
-                npc.create_memory,
-                npc.read_memory,
-                npc.update_memory,
-                npc.delete_memory,
-                npc.search_memories,
-                npc.get_all_memories,
-                npc.archive_old_memories,
-                npc.get_memory_stats
-            ])
-        
-        if npc.db_conn:
-            core_tools.append(npc.query_database)
-        
-        if hasattr(npc, 'tools') and npc.tools:
-            core_tools.extend([func for func in npc.tool_map.values() if callable(func)])
-        
-        if core_tools:
-            tools_schema, tool_map = auto_tools(core_tools)
-            actions.update({
-                f"use_{tool.__name__}": {
-                    "description": f"Use {tool.__name__} capability",
-                    "handler": tool,
-                    "context": lambda **_: f"Available as automated capability",
-                    "output_keys": {"result": {"description": "Tool execution result", "type": "string"}}
-                }
-                for tool in core_tools
-            })
-    
-    if team and hasattr(team, 'npcs') and len(team.npcs) > 1:
-        available_npcs = [name for name in team.npcs.keys() if name != (npc.name if npc else None)]
-        
-        def team_aware_handler(command, extracted_data, **kwargs):
-            if 'team' not in kwargs or kwargs['team'] is None:
-                kwargs['team'] = team
-            return agent_pass_handler(command, extracted_data, **kwargs)
-        
-        actions["pass_to_npc"] = {
-            "description": "Pass request to another NPC - only when task requires their specific expertise",
-            "handler": team_aware_handler,
-            "context": lambda npc=npc, team=team, **_: (
-                f"Available NPCs: {', '.join(available_npcs)}. "
-                f"Only pass when you genuinely cannot complete the task."
-            ),
-            "output_keys": {
-                "target_npc": {
-                    "description": "Name of the NPC to pass the request to",
-                    "type": "string"
-                }
-            }
-        }
-    
-    return actions
 def extract_jinx_inputs(args: List[str], jinx: Jinx) -> Dict[str, Any]:
     print(f"DEBUG extract_jinx_inputs called with args: {args}")
     print(f"DEBUG jinx.inputs: {jinx.inputs}")
@@ -867,7 +1073,9 @@ class NPC:
             self.npc_directory = None
 
         self.team = team # Store the team reference (can be None)
-        self.jinxs_spec = jinxs or "*" # Store the jinx specification for later loading
+        # Only set jinxs_spec from parameter if it wasn't already set by _load_from_file
+        if not hasattr(self, 'jinxs_spec') or jinxs is not None:
+            self.jinxs_spec = jinxs or "*" # Store the jinx specification for later loading
 
         if tools is not None:
             tools_schema, tool_map = auto_tools(tools)
@@ -933,10 +1141,29 @@ class NPC:
                         print(f"Warning: Jinx '{jinx_item}' not found for NPC '{self.name}' during initial load.")
         
         self.shared_context = {
+            # Data analysis (guac)
             "dataframes": {},
             "current_data": None,
             "computation_results": [],
-            "memories":{}
+            "locals": {},  # Python exec locals for guac mode
+
+            # Memory
+            "memories": {},
+
+            # MCP tools (corca)
+            "mcp_client": None,
+            "mcp_tools": [],
+            "mcp_tool_map": {},
+
+            # Session tracking
+            "session_input_tokens": 0,
+            "session_output_tokens": 0,
+            "session_cost_usd": 0.0,
+            "turn_count": 0,
+
+            # Mode state
+            "current_mode": "agent",
+            "attachments": [],
         }
         
         for key, value in kwargs.items():
@@ -1715,35 +1942,41 @@ class NPC:
             jinja_env=self.jinja_env # Pass the NPC's second-pass Jinja env
         )
         
-        if self.db_conn is not None:
-            self.db_conn.add_jinx_call(
-                triggering_message_id=message_id,
-                conversation_id=conversation_id,
-                jinx_name=jinx_name,
-                jinx_inputs=inputs,
-                jinx_output=result,
-                status="success",
-                error_message=None,
-                duration_ms=None,
-                npc_name=self.name,
-                team_name=team_name,
-            )
+        # Log jinx call if we have a command_history with add_jinx_call method
+        if self.command_history is not None and hasattr(self.command_history, 'add_jinx_call'):
+            try:
+                self.command_history.add_jinx_call(
+                    triggering_message_id=message_id,
+                    conversation_id=conversation_id,
+                    jinx_name=jinx_name,
+                    jinx_inputs=inputs,
+                    jinx_output=result,
+                    status="success",
+                    error_message=None,
+                    duration_ms=None,
+                    npc_name=self.name,
+                    team_name=team_name,
+                )
+            except Exception:
+                pass  # Don't fail jinx execution due to logging error
         return result
     def check_llm_command(self,
-                            command, 
+                            command,
                             messages=None,
                             context=None,
                             team=None,
-                            stream=False):
+                            stream=False,
+                            jinxs=None):
         """Check if a command is for the LLM"""
         if context is None:
             context = self.shared_context
-        
+
         if team:
             self._current_team = team
-        
-        actions = get_npc_action_space(npc=self, team=team)
-        
+
+        # Use provided jinxs or fall back to NPC's own jinxs
+        jinxs_to_use = jinxs if jinxs is not None else self.jinxs_dict
+
         return npy.llm_funcs.check_llm_command(
             command,
             model=self.model,
@@ -1753,7 +1986,7 @@ class NPC:
             messages=self.memory if messages is None else messages,
             context=context,
             stream=stream,
-            actions=actions  
+            jinxs=jinxs_to_use,
         )
     
     def handle_agent_pass(self, 
@@ -2170,13 +2403,9 @@ class Team:
                     if 'file_patterns' in ctx_data:
                         file_cache = self._parse_file_patterns(ctx_data['file_patterns'])
                         self.shared_context['files'] = file_cache
-                    if 'preferences' in ctx_data:
-                        self.preferences = ctx_data['preferences']
-                    else:
-                        self.preferences = []
-                    
+                    # All other keys (including preferences) are treated as generic context
                     for key, item in ctx_data.items():
-                        if key not in ['name', 'mcp_servers', 'databases', 'context', 'file_patterns', 'forenpc', 'model', 'provider', 'api_url', 'env', 'preferences']:
+                        if key not in ['name', 'mcp_servers', 'databases', 'context', 'file_patterns', 'forenpc', 'model', 'provider', 'api_url', 'env']:
                             self.shared_context[key] = item
                 return # Only load the first .ctx file found
         
@@ -2356,146 +2585,98 @@ class Team:
         else:
             return None
 
-    def orchestrate(self, request):
+    def orchestrate(self, request, max_iterations=3):
         """Orchestrate a request through the team"""
-        forenpc = self.get_forenpc() # Now guaranteed to be an NPC object
+        import re
+        from termcolor import colored
+
+        forenpc = self.get_forenpc()
         if not forenpc:
             return {"error": "No forenpc available to coordinate the team"}
-        
-        log_entry(
-            self.name,
-            "orchestration_start",
-            {"request": request}
-        )
-        
-        result = forenpc.check_llm_command(request,
-            context=getattr(self, 'context', {}),
-            team = self, 
-        )
-        
-        while True:
-            completion_prompt= "" 
-            if isinstance(result, dict):
-                self.shared_context["execution_history"].append(result)
-                
-                if result.get("messages") and result.get("npc_name"):
-                    if result["npc_name"] not in self.shared_context["npc_messages"]:
-                        self.shared_context["npc_messages"][result["npc_name"]] = []
-                    self.shared_context["npc_messages"][result["npc_name"]].extend(
-                        result["messages"]
-                    )
-                
-                completion_prompt += f"""Context:
-                    User request '{request}', previous agent
-                    
-                    previous agent returned:
-                    {result.get('output')}
 
-                    
-                Instructions:
+        print(colored(f"[orchestrate] Starting with forenpc={forenpc.name}, team={self.name}", "cyan"))
+        print(colored(f"[orchestrate] Request: {request[:100]}...", "cyan"))
 
-                    Check whether the response is relevant to the user's request.
+        # Filter out 'orchestrate' jinx to prevent infinite recursion
+        jinxs_for_orchestration = {k: v for k, v in forenpc.jinxs_dict.items() if k != 'orchestrate'}
 
-                """
-                if self.npcs is None or len(self.npcs) == 0:
-                    completion_prompt += f"""
-                    The team has no members, so the forenpc must handle the request alone.
-                    """
-                else:
-                    completion_prompt += f"""
-                    
-                    These are all the members of the team: {', '.join(self.npcs.keys())}
-
-                    Therefore, if you are trying to evaluate whether a request was fulfilled relevantly,
-                    consider that requests are made to the forenpc: {forenpc.name}
-                    and that the forenpc must pass those along to the other npcs. 
-                    """
-                completion_prompt += f"""
-
-                Mainly concern yourself with ensuring there are no
-                glaring errors nor fundamental mishaps in the response.
-                Do not consider stylistic hiccups as the answers being
-                irrelevant. By providing responses back to for the user to
-                comment on, they can can more efficiently iterate and resolve any issues by 
-                prompting more clearly.
-                natural language itself is very fuzzy so there will always be some level
-                of misunderstanding, but as long as the response is clearly relevant 
-                to the input request and along the user's intended direction,
-                it is considered relevant.
-                                
-
-                If there is enough information to begin a fruitful conversation with the user, 
-                please consider the request relevant so that we do not
-                arbritarily stall business logic which is more efficiently
-                determined by iterations than through unnecessary pedantry.
-
-                It is more important to get a response to the user
-                than to account for all edge cases, so as long as the response more or less tackles the
-                initial problem to first order, consider it relevant.
-
-                Return a JSON object with:
-                    -'relevant' with boolean value
-                    -'explanation' for irrelevance with quoted citations in your explanation noting why it is irrelevant to user input must be a single string.
-                Return only the JSON object."""
-            
-            completion_check = npy.llm_funcs.get_llm_response(
-                completion_prompt, 
-                model=forenpc.model,
-                provider=forenpc.provider,
-                api_key=forenpc.api_key,
-                api_url=forenpc.api_url,
-                npc=forenpc,
-                format="json"
+        try:
+            result = forenpc.check_llm_command(
+                request,
+                context=getattr(self, 'context', {}),
+                team=self,
+                jinxs=jinxs_for_orchestration,
             )
-            
-            if isinstance(completion_check.get("response"), dict):
-                complete = completion_check["response"].get("relevant", False)
-                explanation = completion_check["response"].get("explanation", "")
-            else:
-                complete = False
-                explanation = "Could not determine completion status"
-            
-            if complete:
-                debrief = npy.llm_funcs.get_llm_response(
-                    f"""Context:
-                    Original request: {request}
-                    Execution history: {self.shared_context['execution_history']}
-
-                    Instructions:
-                    Provide summary of actions taken and recommendations.
-                    Return a JSON object with:
-                    - 'summary': Overview of what was accomplished
-                    - 'recommendations': Suggested next steps
-                    Return only the JSON object.""",
-                    model=forenpc.model,
-                    provider=forenpc.provider,
-                    api_key=forenpc.api_key,
-                    api_url=forenpc.api_url,
-                    npc=forenpc,
-                    format="json"
-                )
-                
-                return {
-                    "debrief": debrief.get("response"),
-                    "output": result.get("output"),
-                    "execution_history": self.shared_context["execution_history"],
-                }
-            else:
-                updated_request = (
-                    request
-                    + "\n\nThe request has not yet been fully completed. "
-                    + explanation
-                    + "\nPlease address only the remaining parts of the request."
-                )
-                print('updating request', updated_request)
-                
-                result = forenpc.check_llm_command(
-                    updated_request,
-                    context=getattr(self, 'context', {}),
-                    stream = False,
-                    team = self
-                    
-                )
+            print(colored(f"[orchestrate] Initial result type={type(result)}", "cyan"))
+            if isinstance(result, dict):
+                print(colored(f"[orchestrate] Result keys={list(result.keys())}", "cyan"))
+                if 'error' in result:
+                    print(colored(f"[orchestrate] Error in result: {result['error']}", "red"))
+                    return result
+        except Exception as e:
+            print(colored(f"[orchestrate] Exception in check_llm_command: {e}", "red"))
+            return {"error": str(e), "output": f"Orchestration failed: {e}"}
+
+        # Check if forenpc mentioned other team members - if so, delegate to them
+        output = ""
+        if isinstance(result, dict):
+            output = result.get('output') or result.get('response') or ""
+
+        print(colored(f"[orchestrate] Output preview: {output[:200] if output else 'EMPTY'}...", "cyan"))
+
+        if output and self.npcs:
+            # Look for @npc_name mentions OR just npc names
+            at_pattern = r'@(\w+)'
+            mentions = re.findall(at_pattern, output)
+
+            # Also check for NPC names mentioned without @ (case insensitive)
+            if not mentions:
+                for npc_name in self.npcs.keys():
+                    if npc_name.lower() != forenpc.name.lower():
+                        if npc_name.lower() in output.lower():
+                            mentions.append(npc_name)
+                            break
+
+            print(colored(f"[orchestrate] Found mentions: {mentions}", "cyan"))
+
+            for mentioned in mentions:
+                mentioned_lower = mentioned.lower()
+                if mentioned_lower in self.npcs and mentioned_lower != forenpc.name:
+                    target_npc = self.npcs[mentioned_lower]
+                    print(colored(f"[orchestrate] Delegating to @{mentioned_lower}", "yellow"))
+
+                    try:
+                        # Execute the request with the target NPC (exclude orchestrate to prevent loops)
+                        target_jinxs = {k: v for k, v in target_npc.jinxs_dict.items() if k != 'orchestrate'}
+                        delegate_result = target_npc.check_llm_command(
+                            request,
+                            context=getattr(self, 'context', {}),
+                            team=self,
+                            jinxs=target_jinxs,
+                        )
+
+                        if isinstance(delegate_result, dict):
+                            delegate_output = delegate_result.get('output') or delegate_result.get('response') or ""
+                            if delegate_output:
+                                output = f"[{mentioned_lower}]: {delegate_output}"
+                                result = delegate_result
+                                print(colored(f"[orchestrate] Got response from {mentioned_lower}", "green"))
+                    except Exception as e:
+                        print(colored(f"[orchestrate] Delegation to {mentioned_lower} failed: {e}", "red"))
+
+                    break  # Only delegate to first mentioned NPC
+
+        if isinstance(result, dict):
+            final_output = output if output else str(result)
+            return {
+                "output": final_output,
+                "result": result,
+            }
+        else:
+            return {
+                "output": str(result),
+                "result": result,
+            }
                 
     def to_dict(self):
         """Convert team to dictionary representation"""