npcpy 1.2.19__py3-none-any.whl → 1.2.21__py3-none-any.whl

npcpy/ft/memory_trainer.py

@@ -3,162 +3,169 @@ try:
     import torch
     import torch.nn as nn
     from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
-except:
-    pass 
-import json
-from typing import List, Dict, Tuple
-import random
 
-class MemoryDataset(Dataset):
-    def __init__(self, examples: List[Dict], tokenizer, max_length=512):
-        self.examples = examples
-        self.tokenizer = tokenizer
-        self.max_length = max_length
-    
-    def __len__(self):
-        return len(self.examples)
-    
-    def __getitem__(self, idx):
-        example = self.examples[idx]
-        
-        
-        text = f"Memory: {example['memory']}\nContext: {example.get('context', '')}"
-        
-        encoding = self.tokenizer(
-            text,
-            truncation=True,
-            padding='max_length',
-            max_length=self.max_length,
-            return_tensors='pt'
-        )
-        
-        return {
-            'input_ids': encoding['input_ids'].flatten(),
-            'attention_mask': encoding['attention_mask'].flatten(),
-            'labels': torch.tensor(example['label'], dtype=torch.long)
-        }
+    import json
+    from typing import List, Dict, Tuple
+    import random
 
-class MemoryTrainer:
-    def __init__(self, model_name="google/gemma-2b", device="cpu"):
-        self.device = device
-        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
-        if self.tokenizer.pad_token is None:
-            self.tokenizer.pad_token = self.tokenizer.eos_token
-        
-        
-        self.model = AutoModelForSequenceClassification.from_pretrained(
-            model_name, 
-            num_labels=3
-        ).to(device)
-    
-    def prepare_training_data(self, approved_memories: List[Dict], 
-                            rejected_memories: List[Dict]) -> List[Dict]:
-        """Prepare training data from memory examples"""
-        examples = []
-        
-        
-        for memory in approved_memories:
-            examples.append({
-                "memory": memory.get("final_memory") or memory.get("initial_memory"),
-                "context": memory.get("context", ""),
-                "label": 1
-            })
-        
-        
-        for memory in rejected_memories:
-            examples.append({
-                "memory": memory.get("initial_memory"),
-                "context": memory.get("context", ""),
-                "label": 0
-            })
-        
-        
-        edited_examples = []
-        for memory in approved_memories[:len(rejected_memories)//2]:
-            if memory.get("final_memory") and memory.get("initial_memory"):
-                
-                edited_examples.append({
+    class MemoryDataset(Dataset):
+        def __init__(self, examples: List[Dict], tokenizer, max_length=512):
+            self.examples = examples
+            self.tokenizer = tokenizer
+            self.max_length = max_length
+        
+        def __len__(self):
+            return len(self.examples)
+        
+        def __getitem__(self, idx):
+            example = self.examples[idx]
+            
+            
+            text = f"Memory: {example['memory']}\nContext: {example.get('context', '')}"
+            
+            encoding = self.tokenizer(
+                text,
+                truncation=True,
+                padding='max_length',
+                max_length=self.max_length,
+                return_tensors='pt'
+            )
+            
+            return {
+                'input_ids': encoding['input_ids'].flatten(),
+                'attention_mask': encoding['attention_mask'].flatten(),
+                'labels': torch.tensor(example['label'], dtype=torch.long)
+            }
+
+    class MemoryTrainer:
+        def __init__(self, model_name="google/gemma-2b", device="cpu"):
+            self.device = device
+            self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+            if self.tokenizer.pad_token is None:
+                self.tokenizer.pad_token = self.tokenizer.eos_token
+            
+            
+            self.model = AutoModelForSequenceClassification.from_pretrained(
+                model_name, 
+                num_labels=3
+            ).to(device)
+        
+        def prepare_training_data(self, approved_memories: List[Dict], 
+                                rejected_memories: List[Dict]) -> List[Dict]:
+            """Prepare training data from memory examples"""
+            examples = []
+            
+            
+            for memory in approved_memories:
+                examples.append({
+                    "memory": memory.get("final_memory") or memory.get("initial_memory"),
+                    "context": memory.get("context", ""),
+                    "label": 1
+                })
+            
+            
+            for memory in rejected_memories:
+                examples.append({
                     "memory": memory.get("initial_memory"),
                     "context": memory.get("context", ""),
-                    "label": 2
+                    "label": 0
                 })
-        
-        examples.extend(edited_examples)
-        random.shuffle(examples)
-        return examples
+            
+            
+            edited_examples = []
+            for memory in approved_memories[:len(rejected_memories)//2]:
+                if memory.get("final_memory") and memory.get("initial_memory"):
+                    
+                    edited_examples.append({
+                        "memory": memory.get("initial_memory"),
+                        "context": memory.get("context", ""),
+                        "label": 2
+                    })
+            
+            examples.extend(edited_examples)
+            random.shuffle(examples)
+            return examples
 
-    def train(self, approved_memories: List[Dict], rejected_memories: List[Dict], 
-              output_dir: str = "./memory_model", epochs: int = 3):
-        """Train the memory classification model"""
-        
-        if len(approved_memories) < 10 or len(rejected_memories) < 10:
-            print("Not enough training data. Need at least 10 approved and 10 rejected memories.")
-            return False
-        
-        training_data = self.prepare_training_data(approved_memories, rejected_memories)
-        
-        
-        split_idx = int(0.8 * len(training_data))
-        train_data = training_data[:split_idx]
-        val_data = training_data[split_idx:]
-        
-        train_dataset = MemoryDataset(train_data, self.tokenizer)
-        val_dataset = MemoryDataset(val_data, self.tokenizer)
-        
-        training_args = TrainingArguments(
-            output_dir=output_dir,
-            num_train_epochs=epochs,
-            per_device_train_batch_size=4,
-            per_device_eval_batch_size=4,
-            warmup_steps=100,
-            weight_decay=0.01,
-            logging_dir='./logs',
-            evaluation_strategy="epoch",
-            save_strategy="epoch",
-            load_best_model_at_end=True,
-        )
-        
-        trainer = Trainer(
-            model=self.model,
-            args=training_args,
-            train_dataset=train_dataset,
-            eval_dataset=val_dataset,
-        )
-        
-        trainer.train()
-        trainer.save_model()
-        self.tokenizer.save_pretrained(output_dir)
-        
-        print(f"Model trained and saved to {output_dir}")
-        return True
+        def train(self, approved_memories: List[Dict], rejected_memories: List[Dict], 
+                output_dir: str = "./memory_model", epochs: int = 3):
+            """Train the memory classification model"""
+            
+            if len(approved_memories) < 10 or len(rejected_memories) < 10:
+                print("Not enough training data. Need at least 10 approved and 10 rejected memories.")
+                return False
+            
+            training_data = self.prepare_training_data(approved_memories, rejected_memories)
+            
+            
+            split_idx = int(0.8 * len(training_data))
+            train_data = training_data[:split_idx]
+            val_data = training_data[split_idx:]
+            
+            train_dataset = MemoryDataset(train_data, self.tokenizer)
+            val_dataset = MemoryDataset(val_data, self.tokenizer)
+            
+            training_args = TrainingArguments(
+                output_dir=output_dir,
+                num_train_epochs=epochs,
+                per_device_train_batch_size=4,
+                per_device_eval_batch_size=4,
+                warmup_steps=100,
+                weight_decay=0.01,
+                logging_dir='./logs',
+                evaluation_strategy="epoch",
+                save_strategy="epoch",
+                load_best_model_at_end=True,
+            )
+            
+            trainer = Trainer(
+                model=self.model,
+                args=training_args,
+                train_dataset=train_dataset,
+                eval_dataset=val_dataset,
+            )
+            
+            trainer.train()
+            trainer.save_model()
+            self.tokenizer.save_pretrained(output_dir)
+            
+            print(f"Model trained and saved to {output_dir}")
+            return True
 
-    def predict_memory_action(self, memory_content: str, context: str = "") -> Tuple[str, float]:
-        """Predict what action to take on a memory"""
-        text = f"Memory: {memory_content}\nContext: {context}"
-        
-        encoding = self.tokenizer(
-            text,
-            truncation=True,
-            padding=True,
-            max_length=512,
-            return_tensors='pt'
-        ).to(self.device)
-        
-        with torch.no_grad():
-            outputs = self.model(**encoding)
-            probabilities = torch.softmax(outputs.logits, dim=-1)
-            predicted_class = torch.argmax(probabilities, dim=-1).item()
-            confidence = probabilities[0][predicted_class].item()
-        
-        actions = {0: "model-rejected", 1: "model-approved", 2: "needs-editing"}
-        return actions[predicted_class], confidence
+        def predict_memory_action(self, memory_content: str, context: str = "") -> Tuple[str, float]:
+            """Predict what action to take on a memory"""
+            text = f"Memory: {memory_content}\nContext: {context}"
+            
+            encoding = self.tokenizer(
+                text,
+                truncation=True,
+                padding=True,
+                max_length=512,
+                return_tensors='pt'
+            ).to(self.device)
+            
+            with torch.no_grad():
+                outputs = self.model(**encoding)
+                probabilities = torch.softmax(outputs.logits, dim=-1)
+                predicted_class = torch.argmax(probabilities, dim=-1).item()
+                confidence = probabilities[0][predicted_class].item()
+            
+            actions = {0: "model-rejected", 1: "model-approved", 2: "needs-editing"}
+            return actions[predicted_class], confidence
 
-    def auto_approve_memory(self, memory_content: str, context: str = "", 
-                          confidence_threshold: float = 0.8) -> Dict:
-        """Auto-approve memory if confidence is high enough"""
-        action, confidence = self.predict_memory_action(memory_content, context)
-        
-        if confidence >= confidence_threshold:
-            return {"action": action, "confidence": confidence, "auto_processed": True}
-        else:
-            return {"action": "pending_approval", "confidence": confidence, "auto_processed": False}
+        def auto_approve_memory(self, memory_content: str, context: str = "", 
+                            confidence_threshold: float = 0.8) -> Dict:
+            """Auto-approve memory if confidence is high enough"""
+            action, confidence = self.predict_memory_action(memory_content, context)
+            
+            if confidence >= confidence_threshold:
+                return {"action": action, "confidence": confidence, "auto_processed": True}
+            else:
+                return {"action": "pending_approval", "confidence": confidence, "auto_processed": False}    
+except:
+    Dataset = None
+    nn = None
+    Trainer = None
+    TrainingArguments = None
+
+    MemoryDataset = None
+    MemoryTrainer = None

npcpy/npc_compiler.py

@@ -1908,7 +1908,8 @@ class Team:
             "dataframes": {},
             "memories": {},          
             "execution_history": [],   
-            "npc_messages": {}                 
+            "npc_messages": {},
+            "context":''       
             }
                 
         if team_path:
@@ -2017,11 +2018,12 @@ class Team:
                         self.databases = ctx_data['databases']
                     else:
                         self.databases = []
-                    if 'context' in ctx_data:
-                        self.context = ctx_data['context']
-                    else:
-                        self.context = ''
-
+                    
+                    base_context = ctx_data.get('context', '')
+                    self.shared_context['context'] = base_context
+                    if 'file_patterns' in ctx_data:
+                        file_cache = self._parse_file_patterns(ctx_data['file_patterns'])
+                        self.shared_context['files'] = file_cache
                     if 'preferences' in ctx_data:
                         self.preferences = ctx_data['preferences']
                     else:
@@ -2031,7 +2033,7 @@ class Team:
                     else:
                         self.forenpc = self.npcs[list(self.npcs.keys())[0]] if self.npcs else None
                     for key, item in ctx_data.items():
-                        if key not in ['name', 'mcp_servers', 'databases', 'context']:
+                        if key not in ['name', 'mcp_servers', 'databases', 'context', 'file_patterns']:
                             self.shared_context[key] = item
                 return ctx_data
         return {}
@@ -2288,6 +2290,85 @@ class Team:
             team.save(team_dir)
             
         return True
+    def _parse_file_patterns(self, patterns_config):
+        """Parse file patterns configuration and load matching files into KV cache"""
+        if not patterns_config:
+            return {}
+        
+        file_cache = {}
+        
+        for pattern_entry in patterns_config:
+            if isinstance(pattern_entry, str):
+                pattern_entry = {"pattern": pattern_entry}
+            
+            pattern = pattern_entry.get("pattern", "")
+            recursive = pattern_entry.get("recursive", False)
+            base_path = pattern_entry.get("base_path", ".")
+            
+            if not pattern:
+                continue
+                
+            base_path = os.path.expanduser(base_path)
+            if not os.path.isabs(base_path):
+                base_path = os.path.join(self.team_path or os.getcwd(), base_path)
+            
+            matching_files = self._find_matching_files(pattern, base_path, recursive)
+            
+            for file_path in matching_files:
+                file_content = self._load_file_content(file_path)
+                if file_content:
+                    relative_path = os.path.relpath(file_path, base_path)
+                    file_cache[relative_path] = file_content
+        
+        return file_cache
+
+    def _find_matching_files(self, pattern, base_path, recursive=False):
+        """Find files matching the given pattern"""
+        matching_files = []
+        
+        if not os.path.exists(base_path):
+            return matching_files
+        
+        if recursive:
+            for root, dirs, files in os.walk(base_path):
+                for filename in files:
+                    if fnmatch.fnmatch(filename, pattern):
+                        matching_files.append(os.path.join(root, filename))
+        else:
+            try:
+                for item in os.listdir(base_path):
+                    item_path = os.path.join(base_path, item)
+                    if os.path.isfile(item_path) and fnmatch.fnmatch(item, pattern):
+                        matching_files.append(item_path)
+            except PermissionError:
+                print(f"Permission denied accessing {base_path}")
+        
+        return matching_files
+
+    def _load_file_content(self, file_path):
+        """Load content from a file with error handling"""
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+        except Exception as e:
+            print(f"Error reading {file_path}: {e}")
+            return None
+
+
+    def _format_parsed_files_context(self, parsed_files):
+        """Format parsed files into context string"""
+        if not parsed_files:
+            return ""
+        
+        context_parts = ["Additional context from files:"]
+        
+        for file_path, content in parsed_files.items():
+            context_parts.append(f"\n--- {file_path} ---")
+            context_parts.append(content)
+            context_parts.append("")
+        
+        return "\n".join(context_parts)
+
 class Pipeline:
     def __init__(self, pipeline_data=None, pipeline_path=None, npc_team=None):
         """Initialize a pipeline from data or file path"""

npcpy/serve.py

@@ -443,7 +443,7 @@ def get_global_settings():
             "embedding_model": "nomic-embed-text",
             "embedding_provider": "ollama",
             "search_provider": "perplexity",
-            "NPCSH_LICENSE_KEY": "",
+            "NPC_STUDIO_LICENSE_KEY": "",
             "default_folder": os.path.expanduser("~/.npcsh/"),
         }
         global_vars = {}
@@ -479,7 +479,7 @@ def get_global_settings():
                         "NPCSH_EMBEDDING_MODEL": "embedding_model",
                         "NPCSH_EMBEDDING_PROVIDER": "embedding_provider",
                         "NPCSH_SEARCH_PROVIDER": "search_provider",
-                        "NPCSH_LICENSE_KEY": "NPCSH_LICENSE_KEY",
+                        "NPC_STUDIO_LICENSE_KEY": "NPC_STUDIO_LICENSE_KEY",
                         "NPCSH_STREAM_OUTPUT": "NPCSH_STREAM_OUTPUT",
                         "NPC_STUDIO_DEFAULT_FOLDER": "default_folder",
                     }
@@ -521,7 +521,7 @@ def save_global_settings():
             "embedding_model": "NPCSH_EMBEDDING_MODEL",
             "embedding_provider": "NPCSH_EMBEDDING_PROVIDER",
             "search_provider": "NPCSH_SEARCH_PROVIDER",
-            "NPCSH_LICENSE_KEY": "NPCSH_LICENSE_KEY",
+            "NPC_STUDIO_LICENSE_KEY": "NPC_STUDIO_LICENSE_KEY",
             "NPCSH_STREAM_OUTPUT": "NPCSH_STREAM_OUTPUT",
             "default_folder": "NPC_STUDIO_DEFAULT_FOLDER",
         }

{npcpy-1.2.19.dist-info → npcpy-1.2.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: npcpy
-Version: 1.2.19
+Version: 1.2.21
 Summary: npcpy is the premier open-source library for integrating LLMs and Agents into python systems.
 Home-page: https://github.com/NPC-Worldwide/npcpy
 Author: Christopher Agostino
@@ -55,6 +55,7 @@ Requires-Dist: kuzu; extra == "local"
 Requires-Dist: chromadb; extra == "local"
 Requires-Dist: diffusers; extra == "local"
 Requires-Dist: torch; extra == "local"
+Requires-Dist: datasets; extra == "local"
 Provides-Extra: yap
 Requires-Dist: pyaudio; extra == "yap"
 Requires-Dist: gtts; extra == "yap"
@@ -74,6 +75,7 @@ Requires-Dist: kuzu; extra == "all"
 Requires-Dist: chromadb; extra == "all"
 Requires-Dist: diffusers; extra == "all"
 Requires-Dist: torch; extra == "all"
+Requires-Dist: datasets; extra == "all"
 Requires-Dist: pyaudio; extra == "all"
 Requires-Dist: gtts; extra == "all"
 Requires-Dist: playsound==1.2.2; extra == "all"
@@ -93,19 +95,15 @@ Dynamic: requires-python
 Dynamic: summary
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy.png" alt="npcpy logo of a solarpunk sign">
+  <a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md"> 
+  <img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
 </p>
 
-
 # npcpy
 
 Welcome to `npcpy`, the core library of the NPC Toolkit that supercharges natural language processing pipelines and agent tooling. `npcpy` is a flexible framework for building state-of-the-art applications and conducting novel research with LLMs.
 
 
-<p align="center">
-  <a href= "https://github.com/cagostino/npcpy/blob/main/docs/npcpy.md"> 
-  <img src="https://raw.githubusercontent.com/cagostino/npcpy/main/npcpy/npc-python.png" alt="npc-python logo" width=250></a>
-</p>
 
 
 Here is an example for getting responses for a particular agent:

{npcpy-1.2.19.dist-info → npcpy-1.2.21.dist-info}/RECORD

@@ -1,10 +1,10 @@
 npcpy/__init__.py,sha256=9imxFtK74_6Rw9rz0kyMnZYl_voPb569tkTlYLt0Urg,131
 npcpy/llm_funcs.py,sha256=tvcZuQEcIUJClwEJQXBF6ArEVjSuXt1jAcZOcnYWsVQ,85101
 npcpy/main.py,sha256=RWoRIj6VQLxKdOKvdVyaq2kwG35oRpeXPvp1CAAoG-w,81
-npcpy/npc_compiler.py,sha256=BpNlrjwkxhERTrFeFtvv9CUqzULoD2JQuEwRtqwQHLY,92107
+npcpy/npc_compiler.py,sha256=10vu-9WUmlVzaFM_hMJH28iNS1IJXQP3Rb5RT1rZmpA,95326
 npcpy/npc_sysenv.py,sha256=lPYlKM_TeR4l4-Jcgiqq3CCge8b2oFHdfISD4L_G7eo,30308
 npcpy/npcs.py,sha256=eExuVsbTfrRobTRRptRpDm46jCLWUgbvy4_U7IUQo-c,744
-npcpy/serve.py,sha256=RPYT3ZMu-OmO6dg3Ss04wZQNlZQive5cm2UPUbtToV0,100271
+npcpy/serve.py,sha256=O1dxISi0nQ6jsSOSxBXsULgkltnIcyBS6Z0AjfWmuXA,100296
 npcpy/tools.py,sha256=A5_oVmZkzGnI3BI-NmneuxeXQq-r29PbpAZP4nV4jrc,5303
 npcpy/data/__init__.py,sha256=1tcoChR-Hjn905JDLqaW9ElRmcISCTJdE7BGXPlym2Q,642
 npcpy/data/audio.py,sha256=goon4HfsYgx0bI-n1lhkrzWPrJoejJlycXcB0P62pyk,11280
@@ -17,7 +17,7 @@ npcpy/data/web.py,sha256=ARGoVKUlQmaiX0zJbSvvFmRCwOv_Z7Pcan9c5GxYObQ,5117
 npcpy/ft/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/ft/diff.py,sha256=R3Qo6v0-6M1iI0wiXhUzyuYI2ja0q_0i9bE0z3coxzU,28
 npcpy/ft/ge.py,sha256=my5LtGyVTT40V0i1h9FR-tFFA1FHSga-PeCCgUX1UUI,61
-npcpy/ft/memory_trainer.py,sha256=410BtNj308c7V45E809ILbDjCnVRy7n0mdIp2DKOCNY,5904
+npcpy/ft/memory_trainer.py,sha256=QZPznxEEwXbOGroHdMUMa5xpqlNwgV6nqOazI2xgrnQ,6635
 npcpy/ft/rl.py,sha256=l3RUkEJe4b2yB6pildveu2LJymtNq0F17COwf_CCq3U,34
 npcpy/ft/sft.py,sha256=i4ENygRPArbLWN4XZZuBnPWaehs8M-J68JB_mewGJHI,62
 npcpy/gen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -41,8 +41,8 @@ npcpy/work/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 npcpy/work/desktop.py,sha256=F3I8mUtJp6LAkXodsh8hGZIncoads6c_2Utty-0EdDA,2986
 npcpy/work/plan.py,sha256=QyUwg8vElWiHuoS-xK4jXTxxHvkMD3VkaCEsCmrEPQk,8300
 npcpy/work/trigger.py,sha256=P1Y8u1wQRsS2WACims_2IdkBEar-iBQix-2TDWoW0OM,9948
-npcpy-1.2.19.dist-info/licenses/LICENSE,sha256=j0YPvce7Ng9e32zYOu0EmXjXeJ0Nwawd0RA3uSGGH4E,1070
-npcpy-1.2.19.dist-info/METADATA,sha256=dnJPjMvml-0yrEJXUt9g-Rf-UT9mIXuQR3HJZzYgm18,26084
-npcpy-1.2.19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-npcpy-1.2.19.dist-info/top_level.txt,sha256=g1pbSvrOOncB74Bg5-J0Olg4V0A5VzDw-Xz5YObq8BU,6
-npcpy-1.2.19.dist-info/RECORD,,
+npcpy-1.2.21.dist-info/licenses/LICENSE,sha256=j0YPvce7Ng9e32zYOu0EmXjXeJ0Nwawd0RA3uSGGH4E,1070
+npcpy-1.2.21.dist-info/METADATA,sha256=UoIjUU728VK_7CUbvzVqNnXkK8AjPEPwagfXBIdO1ko,26025
+npcpy-1.2.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+npcpy-1.2.21.dist-info/top_level.txt,sha256=g1pbSvrOOncB74Bg5-J0Olg4V0A5VzDw-Xz5YObq8BU,6
+npcpy-1.2.21.dist-info/RECORD,,