npcpy 1.2.37__tar.gz → 1.3.2__tar.gz

{npcpy-1.2.37/npcpy.egg-info → npcpy-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npcpy
-Version: 1.2.37
+Version: 1.3.2
 Summary: npcpy is the premier open-source library for integrating LLMs and Agents into python systems.
 Home-page: https://github.com/NPC-Worldwide/npcpy
 Author: Christopher Agostino
@@ -275,9 +275,8 @@ output = context['research_summary']
             "code": '''
 # Access outputs from previous steps.
 research_summary = context['initial_llm_research']
-# The output of a declarative jinx call (like 'file_reader') is stored under its step name.
-# The actual content we want is the 'output' of the *last step* within that sub-jinx.
-file_summary = context['read_and_process_source_file'].get('output', 'No file summary available.')
+# The file_reader jinx returns its output directly; also keep a fallback of file_raw_content.
+file_summary = context.get('read_and_process_source_file', '') or context.get('file_raw_content', 'No file summary available.')
 
 prompt = f"""Based on the following information:
 1. Comprehensive Research Summary:

{npcpy-1.2.37 → npcpy-1.3.2}/README.md

@@ -179,9 +179,8 @@ output = context['research_summary']
             "code": '''
 # Access outputs from previous steps.
 research_summary = context['initial_llm_research']
-# The output of a declarative jinx call (like 'file_reader') is stored under its step name.
-# The actual content we want is the 'output' of the *last step* within that sub-jinx.
-file_summary = context['read_and_process_source_file'].get('output', 'No file summary available.')
+# The file_reader jinx returns its output directly; also keep a fallback of file_raw_content.
+file_summary = context.get('read_and_process_source_file', '') or context.get('file_raw_content', 'No file summary available.')
 
 prompt = f"""Based on the following information:
 1. Comprehensive Research Summary:

{npcpy-1.2.37 → npcpy-1.3.2}/npcpy/gen/response.py

@@ -346,7 +346,12 @@ def get_ollama_response(
         res = ollama.chat(**api_params, options=options)
         result["raw_response"] = res
 
-        # Extract usage from ollama response
+        if stream:
+            # Return immediately for streaming - don't check 'in' on generator as it consumes it
+            result["response"] = res
+            return result
+
+        # Extract usage from ollama response (only for non-streaming)
         if hasattr(res, 'prompt_eval_count') or 'prompt_eval_count' in res:
             input_tokens = getattr(res, 'prompt_eval_count', None) or res.get('prompt_eval_count', 0) or 0
             output_tokens = getattr(res, 'eval_count', None) or res.get('eval_count', 0) or 0
@@ -354,10 +359,6 @@ def get_ollama_response(
                 "input_tokens": input_tokens,
                 "output_tokens": output_tokens,
             }
-
-        if stream:
-            result["response"] = res
-            return result
         else:
 
             message = res.get("message", {})

{npcpy-1.2.37 → npcpy-1.3.2}/npcpy/npc_compiler.py

@@ -384,7 +384,6 @@ def write_yaml_file(file_path, data):
         print(f"Error writing YAML file {file_path}: {e}")
         return False
 
-
 class Jinx:
     ''' 
     Jinx represents a workflow template with Jinja-rendered steps.
@@ -394,7 +393,8 @@ class Jinx:
     - inputs: list of input parameters
     - description: what the jinx does
     - npc: optional NPC to execute with
-    - steps: list of step definitions with code
+    - steps: list of step definitions with code. This section can now be a Jinja template itself.
+    - file_context: optional list of file patterns to include as context
     
     Execution:
     - Renders Jinja templates in step code with input values
@@ -409,9 +409,13 @@ class Jinx:
         else:
             raise ValueError("Either jinx_data or jinx_path must be provided")
         
-        # Keep a copy for macro expansion, but retain the executable steps by default
+        # _raw_steps will now hold the original, potentially templated, steps definition
         self._raw_steps = list(self.steps)
-        self.steps = list(self._raw_steps)
+        self.steps = [] # Will be populated after first-pass rendering
+        self.parsed_files = {}
+        if self.file_context:
+            self.parsed_files = self._parse_file_patterns(self.file_context)
+
     def _load_from_file(self, path):
         jinx_data = load_yaml_file(path)
         if not jinx_data:
@@ -431,7 +435,8 @@ class Jinx:
         self.inputs = jinx_data.get("inputs", [])
         self.description = jinx_data.get("description", "")
         self.npc = jinx_data.get("npc")
-        self.steps = jinx_data.get("steps", [])
+        self.steps = jinx_data.get("steps", []) # This can now be a Jinja templated list
+        self.file_context = jinx_data.get("file_context", [])
         self._source_path = jinx_data.get("_source_path", None)
 
     def to_tool_def(self) -> Dict[str, Any]:
@@ -463,27 +468,62 @@ class Jinx:
         }
 
     def render_first_pass(
-        self, 
-        jinja_env_for_macros: Environment, 
+        self,
+        jinja_env_for_macros: Environment,
         all_jinx_callables: Dict[str, Callable]
     ):
         """
         Performs the first-pass Jinja rendering on the Jinx's raw steps.
-        This expands nested Jinx calls (e.g., {{ sh(...) }} or 
-        engine: jinx_name) but preserves runtime variables 
-        (e.g., {{ command_var }}).
-        
-        Args:
-            jinja_env_for_macros: The Jinja Environment configured with 
-                Jinx callables in its globals.
-            all_jinx_callables: A dictionary of Jinx names to their 
-                callable functions (from create_jinx_callable).
+        This expands Jinja control flow (for, if) to generate step structures,
+        then expands nested Jinx calls (e.g., {{ sh(...) }} or engine: jinx_name)
+        and inline macros.
         """
-        rendered_steps_output = []
+        # Check if steps are already parsed dicts (common case when loaded from YAML)
+        # If so, skip the YAML string join/parse cycle and use them directly
+        if self._raw_steps and isinstance(self._raw_steps[0], dict):
+            structurally_expanded_steps = list(self._raw_steps)
+        else:
+            # 1. Join the list of raw steps (which are individual YAML lines) into a single string.
+            #    This single string is the complete Jinja template for the 'steps' section.
+            raw_steps_template_string = "\n".join(self._raw_steps)
 
-        for raw_step in self._raw_steps:
+            # 2. Render this single string as a Jinja template.
+            #    Jinja will now process the {% for %} and {% if %} directives,
+            #    dynamically generating the YAML structure.
+            try:
+                steps_template = jinja_env_for_macros.from_string(raw_steps_template_string)
+                # Pass globals (like num_tasks, include_greeting from Jinx inputs)
+                # to the Jinja rendering context for structural templating.
+                rendered_steps_yaml_string = steps_template.render(**jinja_env_for_macros.globals)
+            except Exception as e:
+                # In a real Jinx, this would go to a proper logger.
+                # For this context, we handle the error gracefully.
+                # self._log_debug(f"Warning: Error during first-pass templating of Jinx '{self.jinx_name}' steps YAML: {e}")
+                self.steps = list(self._raw_steps) # Fallback to original raw steps
+                return
+
+            # 3. Parse the rendered YAML string back into a list of step dictionaries.
+            #    This step will now correctly interpret the YAML structure generated by Jinja.
+            try:
+                structurally_expanded_steps = yaml.safe_load(rendered_steps_yaml_string)
+                if not isinstance(structurally_expanded_steps, list):
+                    # Handle cases where the rendered YAML might be empty or not a list
+                    if structurally_expanded_steps is None:
+                        structurally_expanded_steps = []
+                    else:
+                        raise ValueError(f"Rendered steps YAML did not result in a list: {type(structurally_expanded_steps)}")
+            except Exception as e:
+                # self._log_debug(f"Warning: Error re-parsing structurally expanded steps YAML for Jinx '{self.jinx_name}': {e}")
+                self.steps = list(self._raw_steps) # Fallback
+                return
+
+        # 4. Now, iterate through these `structurally_expanded_steps` to expand
+        # declarative Jinx calls (engine: jinx_name) and inline macros.
+        # This is the second phase of the first-pass rendering.
+        final_rendered_steps = []
+        for raw_step in structurally_expanded_steps:
             if not isinstance(raw_step, dict):
-                rendered_steps_output.append(raw_step)
+                final_rendered_steps.append(raw_step)
                 continue
 
             engine_name = raw_step.get('engine')
@@ -502,51 +542,58 @@ class Jinx:
                     expanded_steps = yaml.safe_load(expanded_yaml_string)
                     
                     if isinstance(expanded_steps, list):
-                        rendered_steps_output.extend(expanded_steps)
-                        
+                        final_rendered_steps.extend(expanded_steps)
                     elif expanded_steps is not None:
-                        rendered_steps_output.append(expanded_steps)
-                        
+                        final_rendered_steps.append(expanded_steps)
                 except Exception as e:
-                    print(
-                        f"Warning: Error expanding Jinx '{engine_name}' "
-                        f"within Jinx '{self.jinx_name}' "
-                        f"(declarative): {e}"
-                    )
-                    rendered_steps_output.append(raw_step)
-            # Skip rendering for python/bash engine steps - preserve runtime variables
+                    # self._log_debug(
+                    #     f"Warning: Error expanding Jinx '{engine_name}' "
+                    #     f"within Jinx '{self.jinx_name}' "
+                    #     f"(declarative): {e}"
+                    # )
+                    final_rendered_steps.append(raw_step)
+            # For python/bash engine steps, only inline macro expansion happens in the next block.
+            # The code content itself is preserved for runtime Jinja rendering.
             elif raw_step.get('engine') in ['python', 'bash']:
-                rendered_steps_output.append(raw_step)
+                processed_step = {}
+                for key, value in raw_step.items():
+                    if isinstance(value, str):
+                        try:
+                            template = jinja_env_for_macros.from_string(value)
+                            # Render with empty context for inline macros/static values
+                            rendered_value = template.render({})
+                            try:
+                                loaded_value = yaml.safe_load(rendered_value)
+                                processed_step[key] = loaded_value
+                            except yaml.YAMLError:
+                                processed_step[key] = rendered_value
+                        except Exception as e:
+                            # self._log_debug(f"Warning: Error during first-pass rendering of Jinx '{self.jinx_name}' step field '{key}' (inline macro): {e}")
+                            processed_step[key] = value
+                    else:
+                        processed_step[key] = value
+                final_rendered_steps.append(processed_step)
             else:
-                # For other steps, do first-pass rendering (inline macro expansion)
+                # For other steps (e.g., custom engines, or just data), perform inline macro expansion
                 processed_step = {}
                 for key, value in raw_step.items():
                     if isinstance(value, str):
                         try:
-                            template = jinja_env_for_macros.from_string(
-                                value
-                            )
+                            template = jinja_env_for_macros.from_string(value)
                             rendered_value = template.render({})
-                            
                             try:
-                                loaded_value = yaml.safe_load(
-                                    rendered_value
-                                )
+                                loaded_value = yaml.safe_load(rendered_value)
                                 processed_step[key] = loaded_value
                             except yaml.YAMLError:
                                 processed_step[key] = rendered_value
                         except Exception as e:
-                            print(
-                                f"Warning: Error during first-pass "
-                                f"rendering of Jinx '{self.jinx_name}' "
-                                f"step field '{key}' (inline macro): {e}"
-                            )
+                            # self._log_debug(f"Warning: Error during first-pass rendering of Jinx '{self.jinx_name}' step field '{key}' (inline macro): {e}")
                             processed_step[key] = value
                     else:
                         processed_step[key] = value
-                rendered_steps_output.append(processed_step)
+                final_rendered_steps.append(processed_step)
         
-        self.steps = rendered_steps_output
+        self.steps = final_rendered_steps
 
     def execute(self,
                 input_values: Dict[str, Any],
@@ -575,6 +622,11 @@ class Jinx:
             "messages": messages,
             "npc": active_npc
         })
+        
+        # Add parsed file content to the context
+        if self.parsed_files:
+            context['file_context'] = self._format_parsed_files_context(self.parsed_files)
+            context['files'] = self.parsed_files # Also make raw dict available
 
         for i, step in enumerate(self.steps):
             context = self._execute_step(
@@ -585,9 +637,19 @@ class Jinx:
                 messages=messages,
                 extra_globals=extra_globals
             )
+            # If an error occurred in a step, propagate it and stop execution
+            if "error" in context.get("output", ""):
+                self._log_debug(f"DEBUG: Jinx '{self.jinx_name}' execution stopped due to error in step '{step.get('name', 'unnamed_step')}': {context['output']}")
+                break
 
         return context
 
+    def _log_debug(self, msg: str):
+        """Helper for logging debug messages to a file."""
+        log_file_path = os.path.expanduser("~/jinx_debug_log.txt")
+        with open(log_file_path, "a") as f:
+            f.write(f"[{datetime.now().isoformat()}] {msg}\n")
+
     def _execute_step(self,
                   step: Dict[str, Any],
                   context: Dict[str, Any],
@@ -596,40 +658,31 @@ class Jinx:
                   messages: Optional[List[Dict[str, str]]] = None,
                   extra_globals: Optional[Dict[str, Any]] = None):
         
-        def _log_debug(msg):
-            log_file_path = os.path.expanduser("~/jinx_debug_log.txt")
-            with open(log_file_path, "a") as f:
-                f.write(f"[{datetime.now().isoformat()}] {msg}\n")
-
         code_content = step.get("code", "")
         step_name = step.get("name", "unnamed_step")
         step_npc = step.get("npc")
         
         active_npc = step_npc if step_npc else npc
         
+        # Second pass Jinja rendering: render the step's code with the current runtime context
         try:
             template = jinja_env.from_string(code_content)
             rendered_code = template.render(**context)
         except Exception as e:
-            _log_debug(
-                f"Error rendering template for step {step_name} "
-                f"(second pass): {e}"
+            error_msg = (
+                f"Error rendering template for step '{step_name}' "
+                f"(second pass): {type(e).__name__}: {e}"
             )
-            rendered_code = code_content
+            context['output'] = error_msg
+            self._log_debug(error_msg)
+            return context
         
-        _log_debug(f"rendered jinx code: {rendered_code}")
-        _log_debug(
-            f"DEBUG: Before exec - rendered_code: {rendered_code}"
-        )
-        _log_debug(
-            f"DEBUG: Before exec - context['output'] before step: "
-            f"{context.get('output')}"
-        )
+        self._log_debug(f"DEBUG: Executing step '{step_name}' with rendered code: {rendered_code}")
 
         exec_globals = {
             "__builtins__": __builtins__,
             "npc": active_npc,
-            "context": context,
+            "context": context, # Pass context by reference
             "math": math, 
             "random": random, 
             "datetime": datetime,
@@ -653,43 +706,27 @@ class Jinx:
         if extra_globals:
             exec_globals.update(extra_globals)
         
-        exec_locals = {}
-        
+        exec_locals = {} # Locals for this specific exec call
+
         try:
             exec(rendered_code, exec_globals, exec_locals)
         except Exception as e:
             error_msg = (
-                f"Error executing step {step_name}: "
+                f"Error executing step '{step_name}': "
                 f"{type(e).__name__}: {e}"
             )
             context['output'] = error_msg
-            _log_debug(error_msg)
+            self._log_debug(error_msg)
             return context
         
-        _log_debug(f"DEBUG: After exec - exec_locals: {exec_locals}")
-        _log_debug(
-            f"DEBUG: After exec - 'output' in exec_locals: "
-            f"{'output' in exec_locals}"
-        )
-
+        # Update the main context with any variables set in exec_locals
         context.update(exec_locals)
         
-        _log_debug(
-            f"DEBUG: After context.update(exec_locals) - "
-            f"context['output']: {context.get('output')}"
-        )
-        _log_debug(f"context after jinx ex: {context}")
-
         if "output" in exec_locals:
             outp = exec_locals["output"]
             context["output"] = outp
             context[step_name] = outp
 
-            _log_debug(
-                f"DEBUG: Inside 'output' in exec_locals block - "
-                f"context['output']: {context.get('output')}"
-            )
-
             if messages is not None:
                 messages.append({
                     'role':'assistant', 
@@ -702,13 +739,95 @@ class Jinx:
         
         return context
         
+    def _parse_file_patterns(self, patterns_config):
+        """Parse file patterns configuration and load matching files into KV cache"""
+        if not patterns_config:
+            return {}
+        
+        file_cache = {}
+        
+        for pattern_entry in patterns_config:
+            if isinstance(pattern_entry, str):
+                pattern_entry = {"pattern": pattern_entry}
+            
+            pattern = pattern_entry.get("pattern", "")
+            recursive = pattern_entry.get("recursive", False)
+            base_path = pattern_entry.get("base_path", ".")
+            
+            if not pattern:
+                continue
+                
+            # Resolve base_path relative to jinx's source path or current working directory
+            if self._source_path:
+                base_path = os.path.join(os.path.dirname(self._source_path), base_path)
+            base_path = os.path.expanduser(base_path)
+            
+            if not os.path.isabs(base_path):
+                base_path = os.path.join(os.getcwd(), base_path)
+            
+            matching_files = self._find_matching_files(pattern, base_path, recursive)
+            
+            for file_path in matching_files:
+                file_content = self._load_file_content(file_path)
+                if file_content:
+                    relative_path = os.path.relpath(file_path, base_path)
+                    file_cache[relative_path] = file_content
+        
+        return file_cache
+
+    def _find_matching_files(self, pattern, base_path, recursive=False):
+        """Find files matching the given pattern"""
+        matching_files = []
+        
+        if not os.path.exists(base_path):
+            return matching_files
+        
+        if recursive:
+            for root, dirs, files in os.walk(base_path):
+                for filename in files:
+                    if fnmatch.fnmatch(filename, pattern):
+                        matching_files.append(os.path.join(root, filename))
+        else:
+            try:
+                for item in os.listdir(base_path):
+                    item_path = os.path.join(base_path, item)
+                    if os.path.isfile(item_path) and fnmatch.fnmatch(item, pattern):
+                        matching_files.append(item_path)
+            except PermissionError:
+                print(f"Permission denied accessing {base_path}")
+        
+        return matching_files
+
+    def _load_file_content(self, file_path):
+        """Load content from a file with error handling"""
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            print(f"Error reading {file_path}: {e}")
+            return None
+
+    def _format_parsed_files_context(self, parsed_files):
+        """Format parsed files into context string"""
+        if not parsed_files:
+            return ""
+        
+        context_parts = ["Additional context from files:"]
+        
+        for file_path, content in parsed_files.items():
+            context_parts.append(f"\n--- {file_path} ---")
+            context_parts.append(content)
+            context_parts.append("")
+        
+        return "\n".join(context_parts)
 
     def to_dict(self):
         result = {
             "jinx_name": self.jinx_name,
             "description": self.description,
             "inputs": self.inputs,
-            "steps": self._raw_steps 
+            "steps": self._raw_steps, # Save the original raw steps, which might be templated
+            "file_context": self.file_context
         }
         
         if self.npc:
@@ -754,6 +873,7 @@ class Jinx:
                 "jinx_name": name,
                 "description": doc.strip(),
                 "inputs": inputs,
+                "file_context": [],
                 "steps": [
                     {
                         "name": "mcp_function_call",
@@ -775,6 +895,8 @@ output = {mcp_tool.__module__}.{name}(
         except: 
             pass
 
+
+
 def load_jinxs_from_directory(directory):
     """Load all jinxs from a directory recursively"""
     jinxs = []

{npcpy-1.2.37 → npcpy-1.3.2}/npcpy/serve.py

@@ -3901,26 +3901,41 @@ def stream():
                 print('.', end="", flush=True)
                 dot_count += 1
                 if "hf.co" in model or provider == 'ollama' and 'gpt-oss' not in model:
-                    chunk_content = response_chunk["message"]["content"] if "message" in response_chunk and "content" in response_chunk["message"] else ""
-                    if "message" in response_chunk and "tool_calls" in response_chunk["message"]:
-                        for tool_call in response_chunk["message"]["tool_calls"]:
-                            if "id" in tool_call:
-                                tool_call_data["id"] = tool_call["id"]
-                            if "function" in tool_call:
-                                if "name" in tool_call["function"]:
-                                    tool_call_data["function_name"] = tool_call["function"]["name"]
-                                if "arguments" in tool_call["function"]:
-                                    arg_val = tool_call["function"]["arguments"]
+                    # Ollama returns ChatResponse objects - support both attribute and dict access
+                    msg = getattr(response_chunk, "message", None) or response_chunk.get("message", {}) if hasattr(response_chunk, "get") else {}
+                    chunk_content = getattr(msg, "content", None) or (msg.get("content") if hasattr(msg, "get") else "") or ""
+                    # Extract Ollama thinking/reasoning tokens
+                    thinking_content = getattr(msg, "thinking", None) or (msg.get("thinking") if hasattr(msg, "get") else None)
+                    # Handle tool calls with robust attribute/dict access
+                    tool_calls = getattr(msg, "tool_calls", None) or (msg.get("tool_calls") if hasattr(msg, "get") else None)
+                    if tool_calls:
+                        for tool_call in tool_calls:
+                            tc_id = getattr(tool_call, "id", None) or (tool_call.get("id") if hasattr(tool_call, "get") else None)
+                            if tc_id:
+                                tool_call_data["id"] = tc_id
+                            tc_func = getattr(tool_call, "function", None) or (tool_call.get("function") if hasattr(tool_call, "get") else None)
+                            if tc_func:
+                                tc_name = getattr(tc_func, "name", None) or (tc_func.get("name") if hasattr(tc_func, "get") else None)
+                                if tc_name:
+                                    tool_call_data["function_name"] = tc_name
+                                tc_args = getattr(tc_func, "arguments", None) or (tc_func.get("arguments") if hasattr(tc_func, "get") else None)
+                                if tc_args:
+                                    arg_val = tc_args
                                     if isinstance(arg_val, dict):
                                         arg_val = json.dumps(arg_val)
                                     tool_call_data["arguments"] += arg_val
                     if chunk_content:
                         complete_response.append(chunk_content)
+                    # Extract other fields with robust access
+                    created_at = getattr(response_chunk, "created_at", None) or (response_chunk.get("created_at") if hasattr(response_chunk, "get") else None)
+                    model_name = getattr(response_chunk, "model", None) or (response_chunk.get("model") if hasattr(response_chunk, "get") else model)
+                    msg_role = getattr(msg, "role", None) or (msg.get("role") if hasattr(msg, "get") else "assistant")
+                    done_reason = getattr(response_chunk, "done_reason", None) or (response_chunk.get("done_reason") if hasattr(response_chunk, "get") else None)
                     chunk_data = {
                         "id": None, "object": None,
-                        "created": response_chunk["created_at"] or datetime.datetime.now(),
-                        "model": response_chunk["model"],
-                        "choices": [{"index": 0, "delta": {"content": chunk_content, "role": response_chunk["message"]["role"]}, "finish_reason": response_chunk.get("done_reason")}]
+                        "created": created_at or datetime.datetime.now(),
+                        "model": model_name,
+                        "choices": [{"index": 0, "delta": {"content": chunk_content, "role": msg_role, "reasoning_content": thinking_content}, "finish_reason": done_reason}]
                     }
                     yield f"data: {json.dumps(chunk_data)}\n\n"
                 else:

{npcpy-1.2.37 → npcpy-1.3.2/npcpy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npcpy
-Version: 1.2.37
+Version: 1.3.2
 Summary: npcpy is the premier open-source library for integrating LLMs and Agents into python systems.
 Home-page: https://github.com/NPC-Worldwide/npcpy
 Author: Christopher Agostino
@@ -275,9 +275,8 @@ output = context['research_summary']
             "code": '''
 # Access outputs from previous steps.
 research_summary = context['initial_llm_research']
-# The output of a declarative jinx call (like 'file_reader') is stored under its step name.
-# The actual content we want is the 'output' of the *last step* within that sub-jinx.
-file_summary = context['read_and_process_source_file'].get('output', 'No file summary available.')
+# The file_reader jinx returns its output directly; also keep a fallback of file_raw_content.
+file_summary = context.get('read_and_process_source_file', '') or context.get('file_raw_content', 'No file summary available.')
 
 prompt = f"""Based on the following information:
 1. Comprehensive Research Summary:

{npcpy-1.2.37 → npcpy-1.3.2}/setup.py

@@ -83,7 +83,7 @@ extra_files = package_files("npcpy/npc_team/")
 
 setup(
     name="npcpy",
-    version="1.2.37",
+    version="1.3.2",
     packages=find_packages(exclude=["tests*"]),
     install_requires=base_requirements,  
     extras_require={