npcpy 1.3.19__py3-none-any.whl → 1.3.21__py3-none-any.whl

npcpy/data/web.py

@@ -24,22 +24,118 @@ except:
 
 
 
-def search_exa(query:str, 
-               api_key:str = None, 
+def search_exa(query:str,
+               api_key:str = None,
                top_k = 5,
                **kwargs):
     from exa_py import Exa
     if api_key is None:
-        api_key = os.environ.get('EXA_API_KEY') 
+        api_key = os.environ.get('EXA_API_KEY')
     exa = Exa(api_key)
 
     results = exa.search_and_contents(
-        query, 
-        text=True   
+        query,
+        text=True
     )
     return results.results[0:top_k]
 
 
+def search_searxng(query: str, num_results: int = 5, instance_url: str = None):
+    """Search using SearXNG public instances."""
+    instances = [instance_url] if instance_url else [
+        os.environ.get('SEARXNG_URL'),
+        'https://search.sapti.me',
+        'https://searx.work',
+        'https://search.ononoki.org',
+    ]
+    instances = [i for i in instances if i]
+
+    for instance in instances:
+        try:
+            response = requests.get(
+                f"{instance}/search",
+                params={'q': query, 'format': 'json', 'categories': 'general'},
+                headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0'},
+                timeout=10
+            )
+            if response.status_code == 200:
+                data = response.json()
+                results = []
+                for r in data.get('results', [])[:num_results]:
+                    results.append({
+                        'title': r.get('title', ''),
+                        'link': r.get('url', ''),
+                        'content': r.get('content', '')
+                    })
+                if results:
+                    return results
+        except Exception as e:
+            print(f"SearXNG {instance} failed: {e}")
+            continue
+    return []
+
+
+def search_startpage(query: str, num_results: int = 5):
+    """Search using Startpage (scraping)."""
+    try:
+        response = requests.post(
+            'https://www.startpage.com/sp/search',
+            data={'query': query, 'cat': 'web'},
+            headers={
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+                'Accept': 'text/html',
+            },
+            timeout=10
+        )
+        if response.status_code == 200:
+            soup = BeautifulSoup(response.text, 'html.parser')
+            results = []
+            for item in soup.select('.result')[:num_results]:
+                title_el = item.select_one('h2') or item.select_one('a')
+                link_el = item.select_one('a[href^="http"]')
+                desc_el = item.select_one('p') or item.select_one('.description')
+                if link_el:
+                    results.append({
+                        'title': title_el.get_text(strip=True) if title_el else '',
+                        'link': link_el.get('href', ''),
+                        'content': desc_el.get_text(strip=True) if desc_el else ''
+                    })
+            return results
+    except Exception as e:
+        print(f"Startpage search failed: {e}")
+    return []
+
+
+def search_brave(query: str, num_results: int = 5, api_key: str = None):
+    """Search using Brave Search API."""
+    if api_key is None:
+        api_key = os.environ.get('BRAVE_API_KEY')
+    if not api_key:
+        print("Brave API key not set")
+        return []
+
+    try:
+        response = requests.get(
+            'https://api.search.brave.com/res/v1/web/search',
+            params={'q': query, 'count': num_results},
+            headers={'X-Subscription-Token': api_key, 'Accept': 'application/json'},
+            timeout=10
+        )
+        if response.status_code == 200:
+            data = response.json()
+            results = []
+            for r in data.get('web', {}).get('results', [])[:num_results]:
+                results.append({
+                    'title': r.get('title', ''),
+                    'link': r.get('url', ''),
+                    'content': r.get('description', '')
+                })
+            return results
+    except Exception as e:
+        print(f"Brave search failed: {e}")
+    return []
+
+
 def search_perplexity(
     query: str,
     api_key: str = None,
@@ -134,10 +230,19 @@ def search_web(
             print("DuckDuckGo search failed: ", e)
             urls = []
             results = []
-    elif provider =='exa':
-        return search_exa(query, api_key=api_key, )
+    elif provider == 'exa':
+        return search_exa(query, api_key=api_key, top_k=num_results)
+
+    elif provider == 'searxng':
+        results = search_searxng(query, num_results=num_results)
+
+    elif provider == 'startpage':
+        results = search_startpage(query, num_results=num_results)
+
+    elif provider == 'brave':
+        results = search_brave(query, num_results=num_results, api_key=api_key)
 
-    elif provider =='google':  
+    elif provider == 'google':
         urls = list(search(query, num_results=num_results))
         
         

npcpy/llm_funcs.py

@@ -242,7 +242,8 @@ def get_llm_response(
     base_model, base_provider, base_api_url = _resolve_model_provider(npc, team, model, provider)
 
     def _run_single(run_model, run_provider, run_npc, run_team, run_context, extra_kwargs):
-        system_message = get_system_message(run_npc, run_team) if run_npc is not None else "You are a helpful assistant."
+        _tool_capable = bool(extra_kwargs.get("tools"))
+        system_message = get_system_message(run_npc, run_team, tool_capable=_tool_capable) if run_npc is not None else "You are a helpful assistant."
         ctx_suffix = _context_suffix(run_context)
         run_messages = _build_messages(messages, system_message, prompt, ctx_suffix)
         return get_litellm_response(
@@ -499,12 +500,10 @@ def handle_request_input(
 
 
 def _get_jinxs(npc, team):
-    """Get available jinxs from npc and team."""
+    """Get available jinxs from npc (already filtered by jinxs_spec)."""
     jinxs = {}
     if npc and hasattr(npc, 'jinxs_dict'):
         jinxs.update(npc.jinxs_dict)
-    if team and hasattr(team, 'jinxs_dict'):
-        jinxs.update(team.jinxs_dict)
     return jinxs
 
 

npcpy/memory/command_history.py

@@ -613,7 +613,11 @@ class CommandHistory:
             Column('tool_results', Text),  # JSON array of tool call results
             Column('parent_message_id', String(50)),  # Links assistant response to parent user message for broadcast grouping
             Column('device_id', String(255)),  # UUID of the device that created this message
-            Column('device_name', String(255))  # Human-readable device name
+            Column('device_name', String(255)),  # Human-readable device name
+            Column('params', Text),  # JSON object for LLM params: temperature, top_p, top_k, max_tokens, etc.
+            Column('input_tokens', Integer),  # Input/prompt token count
+            Column('output_tokens', Integer),  # Output/completion token count
+            Column('cost', String(50))  # Cost in USD (stored as string for precision)
         )
         
         Table('message_attachments', metadata,
@@ -679,13 +683,29 @@ class CommandHistory:
         metadata.create_all(self.engine, checkfirst=True)
         init_kg_schema(self.engine)
 
-        # Add parent_message_id column if it doesn't exist (for broadcast grouping)
+        # Add columns if they don't exist (migrations for existing databases)
         if 'sqlite' in str(self.engine.url):
             with self.engine.begin() as conn:
                 try:
                     conn.execute(text("ALTER TABLE conversation_history ADD COLUMN parent_message_id VARCHAR(50)"))
                 except Exception:
                     pass  # Column already exists
+                try:
+                    conn.execute(text("ALTER TABLE conversation_history ADD COLUMN params TEXT"))
+                except Exception:
+                    pass  # Column already exists
+                try:
+                    conn.execute(text("ALTER TABLE conversation_history ADD COLUMN input_tokens INTEGER"))
+                except Exception:
+                    pass  # Column already exists
+                try:
+                    conn.execute(text("ALTER TABLE conversation_history ADD COLUMN output_tokens INTEGER"))
+                except Exception:
+                    pass  # Column already exists
+                try:
+                    conn.execute(text("ALTER TABLE conversation_history ADD COLUMN cost VARCHAR(50)"))
+                except Exception:
+                    pass  # Column already exists
 
     def _setup_execution_triggers(self):
         if 'sqlite' in str(self.engine.url):
@@ -871,6 +891,10 @@ class CommandHistory:
         parent_message_id=None,
         device_id=None,
         device_name=None,
+        gen_params=None,
+        input_tokens=None,
+        output_tokens=None,
+        cost=None,
     ):
         if isinstance(content, (dict, list)):
             content = json.dumps(content, cls=CustomJSONEncoder)
@@ -880,21 +904,31 @@ class CommandHistory:
             tool_calls = json.dumps(tool_calls, cls=CustomJSONEncoder)
         if tool_results is not None and not isinstance(tool_results, str):
             tool_results = json.dumps(tool_results, cls=CustomJSONEncoder)
+        # Serialize gen_params as JSON
+        gen_params_json = None
+        if gen_params is not None and not isinstance(gen_params, str):
+            gen_params_json = json.dumps(gen_params, cls=CustomJSONEncoder)
+        elif isinstance(gen_params, str):
+            gen_params_json = gen_params
 
         # Normalize directory path for cross-platform compatibility
         normalized_directory_path = normalize_path_for_db(directory_path)
 
+        # Convert cost to string for precision
+        cost_str = str(cost) if cost is not None else None
+
         stmt = """
             INSERT INTO conversation_history
-            (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team, reasoning_content, tool_calls, tool_results, parent_message_id, device_id, device_name)
-            VALUES (:message_id, :timestamp, :role, :content, :conversation_id, :directory_path, :model, :provider, :npc, :team, :reasoning_content, :tool_calls, :tool_results, :parent_message_id, :device_id, :device_name)
+            (message_id, timestamp, role, content, conversation_id, directory_path, model, provider, npc, team, reasoning_content, tool_calls, tool_results, parent_message_id, device_id, device_name, params, input_tokens, output_tokens, cost)
+            VALUES (:message_id, :timestamp, :role, :content, :conversation_id, :directory_path, :model, :provider, :npc, :team, :reasoning_content, :tool_calls, :tool_results, :parent_message_id, :device_id, :device_name, :params, :input_tokens, :output_tokens, :cost)
         """
         params = {
             "message_id": message_id, "timestamp": timestamp, "role": role, "content": content,
             "conversation_id": conversation_id, "directory_path": normalized_directory_path, "model": model,
             "provider": provider, "npc": npc, "team": team, "reasoning_content": reasoning_content,
             "tool_calls": tool_calls, "tool_results": tool_results, "parent_message_id": parent_message_id,
-            "device_id": device_id, "device_name": device_name
+            "device_id": device_id, "device_name": device_name, "params": gen_params_json,
+            "input_tokens": input_tokens, "output_tokens": output_tokens, "cost": cost_str
         }
         with self.engine.begin() as conn:
             conn.execute(text(stmt), params)
@@ -1050,7 +1084,35 @@ class CommandHistory:
         
         with self.engine.begin() as conn:
             conn.execute(text(stmt), params)
-            
+
+    def get_approved_memories_by_scope(self):
+        """Get all approved/edited memories grouped by (npc, team, path) scope."""
+        stmt = """
+            SELECT id, npc, team, directory_path, initial_memory, final_memory, status
+            FROM memory_lifecycle
+            WHERE status IN ('human-approved', 'human-edited')
+            ORDER BY npc, team, directory_path, created_at
+        """
+        rows = self._fetch_all(stmt, {})
+
+        from collections import defaultdict
+        memories_by_scope = defaultdict(list)
+        for row in rows:
+            statement = row.get('final_memory') or row.get('initial_memory')
+            scope_key = (
+                row.get('npc') or 'default',
+                row.get('team') or 'global_team',
+                row.get('directory_path') or os.getcwd()
+            )
+            memories_by_scope[scope_key].append({
+                'id': row.get('id'),
+                'statement': statement,
+                'source_text': '',
+                'type': 'explicit',
+                'generation': 0
+            })
+        return dict(memories_by_scope)
+
     def add_attachment(self, message_id, name, attachment_type, data, size, file_path=None):
         timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         stmt = """
@@ -1239,7 +1301,7 @@ class CommandHistory:
         stmt = """
             SELECT id, message_id, timestamp, role, content, conversation_id,
                     directory_path, model, provider, npc, team,
-                    reasoning_content, tool_calls, tool_results
+                    reasoning_content, tool_calls, tool_results, parent_message_id, params
             FROM conversation_history WHERE conversation_id = :conversation_id
             ORDER BY timestamp ASC
         """
@@ -1260,6 +1322,11 @@ class CommandHistory:
                     message_dict["tool_results"] = json.loads(message_dict["tool_results"])
                 except (json.JSONDecodeError, TypeError):
                     pass
+            if message_dict.get("params"):
+                try:
+                    message_dict["params"] = json.loads(message_dict["params"])
+                except (json.JSONDecodeError, TypeError):
+                    pass
         return results
 
     def get_npc_conversation_stats(self, start_date=None, end_date=None) -> pd.DataFrame:
@@ -1468,10 +1535,15 @@ def save_conversation_message(
     skip_if_exists: bool = True,
     device_id: str = None,
     device_name: str = None,
+    gen_params: Dict = None,
+    input_tokens: int = None,
+    output_tokens: int = None,
+    cost: float = None,
     ):
     """
     Saves a conversation message linked to a conversation ID with optional attachments.
-    Now also supports reasoning_content, tool_calls, tool_results, and parent_message_id for broadcast grouping.
+    Now also supports reasoning_content, tool_calls, tool_results, parent_message_id for broadcast grouping,
+    and gen_params for temperature, top_p, top_k, max_tokens, etc.
     If skip_if_exists is True and message_id already exists, skip saving to prevent duplicates.
     """
     if wd is None:
@@ -1504,7 +1576,11 @@ def save_conversation_message(
         tool_results=tool_results,
         parent_message_id=parent_message_id,
         device_id=device_id,
-        device_name=device_name)
+        device_name=device_name,
+        gen_params=gen_params,
+        input_tokens=input_tokens,
+        output_tokens=output_tokens,
+        cost=cost)
 def retrieve_last_conversation(
     command_history: CommandHistory, conversation_id: str
     ) -> str: