mindroot 8.2.0__py3-none-any.whl → 8.4.0__py3-none-any.whl

mindroot/coreplugins/chat/router.py

@@ -220,3 +220,43 @@ async def upload_file(request: Request, log_id: str, file: UploadFile = File(...
         "mime_type": file.content_type
     }
 
+
+from lib.chatlog import count_tokens_for_log_id
+
+@router.get("/chat/{log_id}/tokens")
+async def get_token_count(request: Request, log_id: str):
+    """
+    Get token counts for a chat log identified by log_id, including any delegated tasks.
+    
+    Parameters:
+    - log_id: The log ID to count tokens for
+    
+    Returns:
+    - JSON with token counts or error message if log not found
+    """
+    token_counts = count_tokens_for_log_id(log_id)
+    
+    if token_counts is None:
+        return {"status": "error", "message": f"Chat log with ID {log_id} not found"}
+    
+    return {"status": "ok", "token_counts": token_counts}
+
+@router.get("/chat/{log_id}/tokens")
+async def get_token_count(request: Request, log_id: str):
+    """
+    Get token counts for a chat log identified by log_id, including any delegated tasks.
+    
+    Parameters:
+    - log_id: The log ID to count tokens for
+    
+    Returns:
+    - JSON with token counts or error message if log not found
+    """
+    from lib.token_counter import count_tokens_for_log_id
+    
+    token_counts = count_tokens_for_log_id(log_id)
+    
+    if token_counts is None:
+        return {"status": "error", "message": f"Chat log with ID {log_id} not found"}
+    
+    return {"status": "ok", "token_counts": token_counts}

mindroot/coreplugins/chat/services.py

@@ -267,19 +267,25 @@ async def send_message_to_agent(session_id: str, message: str | List[MessagePart
                 parse_error = False
 
                 results, full_cmds = await agent_.chat_commands(context.current_model, context, messages=context.chat_log.get_recent())
-                for result in results:
-                    if result['cmd'] == 'UNKNOWN':
-                        consecutive_parse_errors += 1
-                        parse_error = True
+                if results is not None:
+                    try:
+                        for result in results:
+                            if result['cmd'] == 'UNKNOWN':
+                                consecutive_parse_errors += 1
+                                parse_error = True
+                    except Exception as e:
+                        pass
 
                 if not parse_error:
                     consecutive_parse_errors = 0
+                else:
+                    await asyncio.sleep(1)
 
                 if consecutive_parse_errors > 6:
                     raise Exception("Too many consecutive parse errors, stopping processing.")
 
                 elif consecutive_parse_errors > 3: 
-                    results.append({"cmd": "UNKNOWN", "args": { "SYSTEM WARNING: Issue valid command list or task processing will be halted. Simplify output."}})
+                    results.append({"cmd": "UNKNOWN", "args": { "SYSTEM WARNING: Issue valid command list or task; processing will be halted. Simplify output."}})
 
                 try:
                     tmp_data3 = { "results": full_cmds }
@@ -340,6 +346,7 @@ async def send_message_to_agent(session_id: str, message: str | List[MessagePart
                     continue_processing = False
             except Exception as e:
                 continue_processing = False
+                await asyncio.sleep(1)
                 trace = traceback.format_exc()
                 msg = str(e)
                 descr = msg + "\n\n" + trace

mindroot/lib/buchatlog.py

@@ -0,0 +1,140 @@
+import os
+import json
+from typing import List, Dict
+import sys
+import traceback
+from mindroot.lib.utils.debug import debug_box
+
+class ChatLog:
+    def __init__(self, log_id=0, agent=None, context_length: int = 4096, user: str = None):
+        self.log_id = log_id
+        self.messages = []
+        self.agent = agent
+        if user is None or user == '' or user == 'None':
+            raise ValueError('User must be provided')
+        # make sure user is string
+        if not isinstance(user, str):
+            # does it have a username?
+            if hasattr(user, 'username'):
+                user = user.username
+            else:
+                # throw an error
+                raise ValueError('ChatLog(): user must be a string or have username field')
+        self.user = user
+        if agent is None or agent == '':
+            raise ValueError('Agent must be provided')
+        self.context_length = context_length
+        self.log_dir = os.environ.get('CHATLOG_DIR', 'data/chat')
+        self.log_dir = os.path.join(self.log_dir, self.user)
+        self.log_dir = os.path.join(self.log_dir, self.agent)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+        self.load_log()
+
+    def _get_log_data(self) -> Dict[str, any]:
+        return {
+            'agent': self.agent,
+            'messages': self.messages
+        }
+
+    def _calculate_message_length(self, message: Dict[str, str]) -> int:
+        return len(json.dumps(message)) // 3
+
+    def add_message(self, message: Dict[str, str]) -> None:
+        if len(self.messages)>0 and self.messages[-1]['role'] == message['role']:
+            print("found repeat role")
+            # check if messasge is str
+            # if so, convert to dict with type 'text':
+            if type(message['content']) == str:
+                message['content'] = [{'type':'text', 'text': message['content']}]
+            elif type(message['content']) == list:
+                for part in message['content']:
+                    if part['type'] == 'image':
+                        print("found image")
+                        self.messages.append(message)
+                        self.save_log()
+                        return
+
+            try:
+                cmd_list = json.loads(self.messages[-1]['content'][0]['text'])
+                if type(cmd_list) != list:
+                    debug_box("1")
+                    cmd_list = [cmd_list]
+                new_json = json.loads(message['content'][0]['text'])
+                if type(new_json) != list:
+                    debug_box("2")
+                    new_json = [new_json]
+                new_cmd_list = cmd_list + new_json
+                debug_box("3")
+                self.messages[-1]['content'] = [{ 'type': 'text', 'text': json.dumps(new_cmd_list) }]
+            except Exception as e:
+                # assume previous mesage was not a command, was a string
+                debug_box("4")
+                print("Could not combine commands, probably normal if user message and previous system output, assuming string", e)
+                if type(self.messages[-1]['content']) == str:
+                    new_msg_text = self.messages[-1]['content'] + message['content'][0]['text']
+                else:
+                    new_msg_text = self.messages[-1]['content'][0]['text'] + message['content'][0]['text']
+                self.messages.append({'role': message['role'], 'content': [{'type': 'text', 'text': new_msg_text}]})
+                #print('could not combine commands. probably normal if user message and previous system output', e)
+                #print(self.messages[-1])
+                #print(message)
+                #raise e
+        else:
+            if len(self.messages)>0:
+                print('roles do not repeat, last message role is ', self.messages[-1]['role'], 'new message role is ', message['role'])
+            debug_box("5")
+            self.messages.append(message)
+        self.save_log()
+
+    def get_history(self) -> List[Dict[str, str]]:
+        return self.messages
+
+    def get_recent(self, max_tokens: int = 4096) -> List[Dict[str, str]]:
+        recent_messages = []
+        total_length = 0
+        #print('returning all messages', self.messages)
+        json_messages = json.dumps(self.messages)
+        return json.loads(json_messages)
+
+        #for message in self.messages:
+        #    message_length = self._calculate_message_length(message)
+        #    if total_length + message_length <= max_tokens:
+        #        recent_messages.append(message)
+        #        total_length += message_length
+        #    else:
+        #        break
+        # 
+        #return recent_messages
+
+    def save_log(self) -> None:
+        log_file = os.path.join(self.log_dir, f'chatlog_{self.log_id}.json')
+        with open(log_file, 'w') as f:
+            json.dump(self._get_log_data(), f, indent=2)
+        
+
+    def load_log(self, log_id = None) -> None:
+        if log_id is None:
+            log_id = self.log_id
+        self.log_id = log_id
+        log_file = os.path.join(self.log_dir, f'chatlog_{log_id}.json')
+        if os.path.exists(log_file):
+            with open(log_file, 'r') as f:
+                log_data = json.load(f)
+                self.agent = log_data.get('agent')
+                self.messages = log_data.get('messages', [])
+            print("Loaded log file at ", log_file)
+            print("Message length: ", len(self.messages))
+        else:
+            print("Could not find log file at ", log_file)
+            self.messages = []
+
+    def delete_log(self) -> None:
+        log_file = os.path.join(self.log_dir, f'chatlog_{self.log_id}.json')
+        if os.path.exists(log_file):
+            os.remove(log_file)
+            print("Deleted log file at ", log_file)
+        else:
+            print("Could not find log file at ", log_file)
+
+

mindroot/lib/buchatlog2.py

@@ -0,0 +1,357 @@
+import os
+import json
+from typing import List, Dict
+import sys
+import traceback
+import re
+import time
+from mindroot.lib.utils.debug import debug_box
+
+class ChatLog:
+    def __init__(self, log_id=0, agent=None, context_length: int = 4096, user: str = None):
+        self.log_id = log_id
+        self.messages = []
+        self.agent = agent
+        if user is None or user == '' or user == 'None':
+            raise ValueError('User must be provided')
+        # make sure user is string
+        if not isinstance(user, str):
+            # does it have a username?
+            if hasattr(user, 'username'):
+                user = user.username
+            else:
+                # throw an error
+                raise ValueError('ChatLog(): user must be a string or have username field')
+        self.user = user
+        if agent is None or agent == '':
+            raise ValueError('Agent must be provided')
+        self.context_length = context_length
+        self.log_dir = os.environ.get('CHATLOG_DIR', 'data/chat')
+        self.log_dir = os.path.join(self.log_dir, self.user)
+        self.log_dir = os.path.join(self.log_dir, self.agent)
+        if not os.path.exists(self.log_dir):
+            os.makedirs(self.log_dir)
+        self.load_log()
+    def _get_log_data(self) -> Dict[str, any]:
+        return {
+            'agent': self.agent,
+            'messages': self.messages
+        }
+
+    def _calculate_message_length(self, message: Dict[str, str]) -> int:
+        return len(json.dumps(message)) // 3
+
+    def add_message(self, message: Dict[str, str]) -> None:
+        if len(self.messages)>0 and self.messages[-1]['role'] == message['role']:
+            print("found repeat role")
+            # check if messasge is str
+            # if so, convert to dict with type 'text':
+            if type(message['content']) == str:
+                message['content'] = [{'type':'text', 'text': message['content']}]
+            elif type(message['content']) == list:
+                for part in message['content']:
+                    if part['type'] == 'image':
+                        print("found image")
+                        self.messages.append(message)
+                        self.save_log()
+                        return
+
+            try:
+                cmd_list = json.loads(self.messages[-1]['content'][0]['text'])
+                if type(cmd_list) != list:
+                    debug_box("1")
+                    cmd_list = [cmd_list]
+                new_json = json.loads(message['content'][0]['text'])
+                if type(new_json) != list:
+                    debug_box("2")
+                    new_json = [new_json]
+                new_cmd_list = cmd_list + new_json
+                debug_box("3")
+                self.messages[-1]['content'] = [{ 'type': 'text', 'text': json.dumps(new_cmd_list) }]
+            except Exception as e:
+                # assume previous mesage was not a command, was a string
+                debug_box("4")
+                print("Could not combine commands, probably normal if user message and previous system output, assuming string", e)
+                if type(self.messages[-1]['content']) == str:
+                    new_msg_text = self.messages[-1]['content'] + message['content'][0]['text']
+                else:
+                    new_msg_text = self.messages[-1]['content'][0]['text'] + message['content'][0]['text']
+                self.messages.append({'role': message['role'], 'content': [{'type': 'text', 'text': new_msg_text}]})
+                #print('could not combine commands. probably normal if user message and previous system output', e)
+                #print(self.messages[-1])
+                #print(message)
+                #raise e
+        else:
+            if len(self.messages)>0:
+                print('roles do not repeat, last message role is ', self.messages[-1]['role'], 'new message role is ', message['role'])
+            debug_box("5")
+            self.messages.append(message)
+        self.save_log()
+
+    def get_history(self) -> List[Dict[str, str]]:
+        return self.messages
+
+    def get_recent(self, max_tokens: int = 4096) -> List[Dict[str, str]]:
+        recent_messages = []
+        total_length = 0
+        #print('returning all messages', self.messages)
+        json_messages = json.dumps(self.messages)
+        return json.loads(json_messages)
+
+        #for message in self.messages:
+        #    message_length = self._calculate_message_length(message)
+        #    if total_length + message_length <= max_tokens:
+        #        recent_messages.append(message)
+        #        total_length += message_length
+        #    else:
+        #        break
+        # 
+        #return recent_messages
+
+    def save_log(self) -> None:
+        log_file = os.path.join(self.log_dir, f'chatlog_{self.log_id}.json')
+        with open(log_file, 'w') as f:
+            json.dump(self._get_log_data(), f, indent=2)
+        
+
+    def load_log(self, log_id = None) -> None:
+        if log_id is None:
+            log_id = self.log_id
+        self.log_id = log_id
+        log_file = os.path.join(self.log_dir, f'chatlog_{log_id}.json')
+        if os.path.exists(log_file):
+            with open(log_file, 'r') as f:
+                log_data = json.load(f)
+                self.agent = log_data.get('agent')
+                self.messages = log_data.get('messages', [])
+            print("Loaded log file at ", log_file)
+            print("Message length: ", len(self.messages))
+        else:
+            print("Could not find log file at ", log_file)
+            self.messages = []
+
+    def delete_log(self) -> None:
+        log_file = os.path.join(self.log_dir, f'chatlog_{self.log_id}.json')
+        if os.path.exists(log_file):
+            os.remove(log_file)
+            print("Deleted log file at ", log_file)
+        else:
+            print("Could not find log file at ", log_file)
+
+    def count_tokens(self) -> Dict[str, int]:
+        """
+        Count tokens in the chat log, providing both sequence totals and cumulative request totals.
+        
+        Returns:
+            Dict with the following keys:
+            - input_tokens_sequence: Total tokens in all user messages
+            - output_tokens_sequence: Total tokens in all assistant messages
+            - input_tokens_total: Cumulative tokens sent to LLM across all requests
+        """
+        # Initialize counters
+        input_tokens_sequence = 0  # Total tokens in all user messages
+        output_tokens_sequence = 0  # Total tokens in all assistant messages
+        input_tokens_total = 0  # Cumulative tokens sent to LLM across all requests
+        
+        # Process each message
+        for i, message in enumerate(self.messages):
+            # Calculate tokens in this message (rough approximation)
+            message_tokens = len(json.dumps(message)) // 4
+            
+            # Add to appropriate sequence counter
+            if message['role'] == 'assistant':
+                output_tokens_sequence += message_tokens
+            else:  # user or system
+                input_tokens_sequence += message_tokens
+            
+            # For each assistant message, calculate the input tokens for that request
+            # (which includes all previous messages)
+            if message['role'] == 'assistant':
+                request_input_tokens = 0
+                for j in range(i):
+                    request_input_tokens += len(json.dumps(self.messages[j])) // 4
+                input_tokens_total += request_input_tokens
+        
+        return {
+            'input_tokens_sequence': input_tokens_sequence,
+            'output_tokens_sequence': output_tokens_sequence,
+            'input_tokens_total': input_tokens_total
+        }
+
+def find_chatlog_file(log_id: str) -> str:
+    """
+    Find a chatlog file by its log_id.
+    
+    Args:
+        log_id: The log ID to search for
+        
+    Returns:
+        The full path to the chatlog file if found, None otherwise
+    """
+    chat_dir = os.environ.get('CHATLOG_DIR', 'data/chat')
+    
+    # Use os.walk to search through all subdirectories
+    for root, dirs, files in os.walk(chat_dir):
+        for file in files:
+            if file == f"chatlog_{log_id}.json":
+                return os.path.join(root, file)
+    
+    return None
+
+def extract_delegate_task_log_ids(messages: List[Dict]) -> List[str]:
+    """
+    Extract log IDs from delegate_task commands in messages.
+    
+    Args:
+        messages: List of chat messages
+        
+    Returns:
+        List of log IDs found in delegate_task commands
+    """
+    log_ids = []
+    
+    for message in messages:
+        if message['role'] == 'assistant':
+            content = message['content']
+            # Handle both string and list content formats
+            if isinstance(content, str):
+                text = content
+            elif isinstance(content, list) and len(content) > 0 and 'text' in content[0]:
+                text = content[0]['text']
+            else:
+                continue
+                
+            # Try to parse as JSON
+            try:
+                commands = json.loads(text)
+                if not isinstance(commands, list):
+                    commands = [commands]
+                    
+                for cmd in commands:
+                    for key, value in cmd.items():
+                        if key == 'delegate_task' and 'log_id' in value:
+                            log_ids.append(value['log_id'])
+            except (json.JSONDecodeError, TypeError, KeyError):
+                # If not JSON, try regex to find log_ids in delegate_task commands
+                matches = re.findall(r'"delegate_task"\s*:\s*{\s*"log_id"\s*:\s*"([^"]+)"', text)
+                log_ids.extend(matches)
+    
+    return log_ids
+
+def get_cache_dir() -> str:
+    """
+    Get the directory for token count cache files.
+    Creates the directory if it doesn't exist.
+    """
+    cache_dir = os.environ.get('TOKEN_CACHE_DIR', 'data/token_cache')
+    if not os.path.exists(cache_dir):
+        os.makedirs(cache_dir)
+    return cache_dir
+
+def get_cache_path(log_id: str) -> str:
+    """
+    Get the path to the cache file for a specific log_id.
+    """
+    cache_dir = get_cache_dir()
+    return os.path.join(cache_dir, f"tokens_{log_id}.json")
+
+def get_cached_token_counts(log_id: str, log_path: str) -> Dict[str, int]:
+    """
+    Get cached token counts if available and valid.
+    
+    Args:
+        log_id: The log ID
+        log_path: Path to the actual log file
+        
+    Returns:
+        Cached token counts if valid, None otherwise
+    """
+    cache_path = get_cache_path(log_id)
+    
+    # If cache doesn't exist, return None
+    if not os.path.exists(cache_path):
+        return None
+    
+    try:
+        # Get modification times
+        log_mtime = os.path.getmtime(log_path)
+        cache_mtime = os.path.getmtime(cache_path)
+        current_time = time.time()
+        
+        # If log was modified after cache was created, cache is invalid
+        if log_mtime > cache_mtime:
+            return None
+        
+        # Don't recalculate sooner than 3 minutes after last calculation
+        if current_time - cache_mtime < 180:  # 3 minutes in seconds
+            with open(cache_path, 'r') as f:
+                return json.load(f)
+                
+        # For logs that haven't been modified in over an hour, consider them "finished"
+        # and use the cache regardless of when it was last calculated
+        if current_time - log_mtime > 3600:  # 1 hour in seconds
+            with open(cache_path, 'r') as f:
+                return json.load(f)
+    
+    except (json.JSONDecodeError, IOError) as e:
+        print(f"Error reading token cache: {e}")
+    
+    return None
+
+def save_token_counts_to_cache(log_id: str, token_counts: Dict[str, int]) -> None:
+    """
+    Save token counts to cache.
+    """
+    cache_path = get_cache_path(log_id)
+    with open(cache_path, 'w') as f:
+        json.dump(token_counts, f)
+
+def count_tokens_for_log_id(log_id: str) -> Dict[str, int]:
+    """
+    Count tokens for a chat log identified by log_id, including any delegated tasks.
+    
+    Args:
+        log_id: The log ID to count tokens for
+        
+    Returns:
+        Dictionary with token counts or None if log not found
+    """
+    # Find the chatlog file
+    chatlog_path = find_chatlog_file(log_id)
+    if not chatlog_path:
+        return None
+    
+    # Check cache first
+    cached_counts = get_cached_token_counts(log_id, chatlog_path)
+    if cached_counts:
+        print(f"Using cached token counts for {log_id}")
+        return cached_counts
+    
+    print(f"Calculating token counts for {log_id}")
+    
+    # Load the chat log
+    with open(chatlog_path, 'r') as f:
+        log_data = json.load(f)
+        
+    # Create a temporary ChatLog instance to count tokens
+    temp_log = ChatLog(log_id=log_id, user="system", agent=log_data.get('agent', 'unknown'))
+    temp_log.messages = log_data.get('messages', [])
+    
+    # Count tokens for this log
+    token_counts = temp_log.count_tokens()
+    
+    # Find delegated task log IDs
+    delegated_log_ids = extract_delegate_task_log_ids(temp_log.messages)
+    
+    # Recursively count tokens for delegated tasks
+    for delegated_id in delegated_log_ids:
+        delegated_counts = count_tokens_for_log_id(delegated_id)
+        if delegated_counts:
+            token_counts['input_tokens_sequence'] += delegated_counts['input_tokens_sequence']
+            token_counts['output_tokens_sequence'] += delegated_counts['output_tokens_sequence']
+            token_counts['input_tokens_total'] += delegated_counts['input_tokens_total']
+    
+    # Save to cache
+    save_token_counts_to_cache(log_id, token_counts)
+    
+    return token_counts

mindroot/lib/chatlog.py

@@ -3,6 +3,8 @@ import json
 from typing import List, Dict
 import sys
 import traceback
+import re
+import time
 from mindroot.lib.utils.debug import debug_box
 
 class ChatLog:
@@ -30,7 +32,6 @@ class ChatLog:
         if not os.path.exists(self.log_dir):
             os.makedirs(self.log_dir)
         self.load_log()
-
     def _get_log_data(self) -> Dict[str, any]:
         return {
             'agent': self.agent,
@@ -137,3 +138,238 @@ class ChatLog:
         else:
             print("Could not find log file at ", log_file)
 
+    def count_tokens(self) -> Dict[str, int]:
+        """
+        Count tokens in the chat log, providing both sequence totals and cumulative request totals.
+        
+        Returns:
+            Dict with the following keys:
+            - input_tokens_sequence: Total tokens in all user messages
+            - output_tokens_sequence: Total tokens in all assistant messages
+            - input_tokens_total: Cumulative tokens sent to LLM across all requests
+        """
+        # Initialize counters
+        input_tokens_sequence = 0  # Total tokens in all user messages
+        output_tokens_sequence = 0  # Total tokens in all assistant messages
+        input_tokens_total = 0  # Cumulative tokens sent to LLM across all requests
+        
+        # Process each message
+        for i, message in enumerate(self.messages):
+            # Calculate tokens in this message (rough approximation)
+            message_tokens = len(json.dumps(message)) // 4
+            
+            # Add to appropriate sequence counter
+            if message['role'] == 'assistant':
+                output_tokens_sequence += message_tokens
+            else:  # user or system
+                input_tokens_sequence += message_tokens
+            
+            # For each assistant message, calculate the input tokens for that request
+            # (which includes all previous messages)
+            if message['role'] == 'assistant':
+                request_input_tokens = 0
+                for j in range(i):
+                    request_input_tokens += len(json.dumps(self.messages[j])) // 4
+                input_tokens_total += request_input_tokens
+        
+        return {
+            'input_tokens_sequence': input_tokens_sequence,
+            'output_tokens_sequence': output_tokens_sequence,
+            'input_tokens_total': input_tokens_total
+        }
+
+def find_chatlog_file(log_id: str) -> str:
+    """
+    Find a chatlog file by its log_id.
+    
+    Args:
+        log_id: The log ID to search for
+        
+    Returns:
+        The full path to the chatlog file if found, None otherwise
+    """
+    chat_dir = os.environ.get('CHATLOG_DIR', 'data/chat')
+    
+    # Use os.walk to search through all subdirectories
+    for root, dirs, files in os.walk(chat_dir):
+        for file in files:
+            if file == f"chatlog_{log_id}.json":
+                return os.path.join(root, file)
+    
+    return None
+
+def extract_delegate_task_log_ids(messages: List[Dict]) -> List[str]:
+    """
+    Extract log IDs from delegate_task commands in messages.
+    
+    Args:
+        messages: List of chat messages
+        
+    Returns:
+        List of log IDs found in delegate_task commands
+    """
+    log_ids = []
+    
+    for message in messages:
+        if message['role'] == 'assistant':
+            content = message['content']
+            # Handle both string and list content formats
+            if isinstance(content, str):
+                text = content
+            elif isinstance(content, list) and len(content) > 0 and 'text' in content[0]:
+                text = content[0]['text']
+            else:
+                continue
+                
+            # Try to parse as JSON
+            try:
+                commands = json.loads(text)
+                if not isinstance(commands, list):
+                    commands = [commands]
+                    
+                for cmd in commands:
+                    for key, value in cmd.items():
+                        if key == 'delegate_task' and 'log_id' in value:
+                            log_ids.append(value['log_id'])
+            except (json.JSONDecodeError, TypeError, KeyError):
+                # If not JSON, try regex to find log_ids in delegate_task commands
+                matches = re.findall(r'"delegate_task"\s*:\s*{\s*"log_id"\s*:\s*"([^"]+)"', text)
+                log_ids.extend(matches)
+    
+    return log_ids
+
+def get_cache_dir() -> str:
+    """
+    Get the directory for token count cache files.
+    Creates the directory if it doesn't exist.
+    """
+    cache_dir = os.environ.get('TOKEN_CACHE_DIR', 'data/token_cache')
+    if not os.path.exists(cache_dir):
+        os.makedirs(cache_dir)
+    return cache_dir
+
+def get_cache_path(log_id: str) -> str:
+    """
+    Get the path to the cache file for a specific log_id.
+    """
+    cache_dir = get_cache_dir()
+    return os.path.join(cache_dir, f"tokens_{log_id}.json")
+
+def get_cached_token_counts(log_id: str, log_path: str) -> Dict[str, int]:
+    """
+    Get cached token counts if available and valid.
+    
+    Args:
+        log_id: The log ID
+        log_path: Path to the actual log file
+        
+    Returns:
+        Cached token counts if valid, None otherwise
+    """
+    cache_path = get_cache_path(log_id)
+    
+    # If cache doesn't exist, return None
+    if not os.path.exists(cache_path):
+        return None
+    
+    try:
+        # Get modification times
+        log_mtime = os.path.getmtime(log_path)
+        cache_mtime = os.path.getmtime(cache_path)
+        current_time = time.time()
+        
+        # If log was modified after cache was created, cache is invalid
+        if log_mtime > cache_mtime:
+            return None
+        
+        # Don't recalculate sooner than 3 minutes after last calculation
+        if current_time - cache_mtime < 180:  # 3 minutes in seconds
+            with open(cache_path, 'r') as f:
+                return json.load(f)
+                
+        # For logs that haven't been modified in over an hour, consider them "finished"
+        # and use the cache regardless of when it was last calculated
+        if current_time - log_mtime > 3600:  # 1 hour in seconds
+            with open(cache_path, 'r') as f:
+                return json.load(f)
+    
+    except (json.JSONDecodeError, IOError) as e:
+        print(f"Error reading token cache: {e}")
+    
+    return None
+
+def save_token_counts_to_cache(log_id: str, token_counts: Dict[str, int]) -> None:
+    """
+    Save token counts to cache.
+    """
+    cache_path = get_cache_path(log_id)
+    with open(cache_path, 'w') as f:
+        json.dump(token_counts, f)
+def count_tokens_for_log_id(log_id: str) -> Dict[str, int]:
+    """
+    Count tokens for a chat log identified by log_id, including any delegated tasks.
+    
+    Args:
+        log_id: The log ID to count tokens for
+        
+    Returns:
+        Dictionary with token counts or None if log not found
+    """
+    # Find the chatlog file
+    chatlog_path = find_chatlog_file(log_id)
+    if not chatlog_path:
+        return None
+    
+    # Check cache first
+    cached_counts = get_cached_token_counts(log_id, chatlog_path)
+    if cached_counts:
+        print(f"Using cached token counts for {log_id}")
+        return cached_counts
+    
+    print(f"Calculating token counts for {log_id}")
+    
+    # Load the chat log
+    with open(chatlog_path, 'r') as f:
+        log_data = json.load(f)
+        
+    # Create a temporary ChatLog instance to count tokens
+    temp_log = ChatLog(log_id=log_id, user="system", agent=log_data.get('agent', 'unknown'))
+    temp_log.messages = log_data.get('messages', [])
+    
+    # Count tokens for this log
+    parent_counts = temp_log.count_tokens()
+    
+    # Create combined counts (starting with parent counts)
+    combined_counts = {
+        'input_tokens_sequence': parent_counts['input_tokens_sequence'],
+        'output_tokens_sequence': parent_counts['output_tokens_sequence'],
+        'input_tokens_total': parent_counts['input_tokens_total']
+    }
+    
+    # Find delegated task log IDs
+    delegated_log_ids = extract_delegate_task_log_ids(temp_log.messages)
+    
+    # Recursively count tokens for delegated tasks
+    for delegated_id in delegated_log_ids:
+        delegated_counts = count_tokens_for_log_id(delegated_id)
+        if delegated_counts:
+            combined_counts['input_tokens_sequence'] += delegated_counts['input_tokens_sequence']
+            combined_counts['output_tokens_sequence'] += delegated_counts['output_tokens_sequence']
+            combined_counts['input_tokens_total'] += delegated_counts['input_tokens_total']
+    
+    # Create final result with both parent and combined counts
+    token_counts = {
+        # Parent session only counts
+        'input_tokens_sequence': parent_counts['input_tokens_sequence'],
+        'output_tokens_sequence': parent_counts['output_tokens_sequence'],
+        'input_tokens_total': parent_counts['input_tokens_total'],
+        # Combined counts (parent + all subtasks)
+        'combined_input_tokens_sequence': combined_counts['input_tokens_sequence'],
+        'combined_output_tokens_sequence': combined_counts['output_tokens_sequence'],
+        'combined_input_tokens_total': combined_counts['input_tokens_total']
+    }
+    
+    # Save to cache
+    save_token_counts_to_cache(log_id, token_counts)
+    
+    return token_counts

mindroot/lib/token_counter.py

@@ -0,0 +1,201 @@
+import os
+import json
+import re
+import time
+from typing import Dict, List
+from mindroot.lib.chatlog import ChatLog
+
+def find_chatlog_file(log_id: str) -> str:
+    """
+    Find a chatlog file by its log_id.
+    
+    Args:
+        log_id: The log ID to search for
+        
+    Returns:
+        The full path to the chatlog file if found, None otherwise
+    """
+    chat_dir = os.environ.get('CHATLOG_DIR', 'data/chat')
+    
+    # Use os.walk to search through all subdirectories
+    for root, dirs, files in os.walk(chat_dir):
+        for file in files:
+            if file == f"chatlog_{log_id}.json":
+                return os.path.join(root, file)
+    
+    return None
+
+def extract_delegate_task_log_ids(messages: List[Dict]) -> List[str]:
+    """
+    Extract log IDs from delegate_task commands in messages.
+    
+    Args:
+        messages: List of chat messages
+        
+    Returns:
+        List of log IDs found in delegate_task commands
+    """
+    log_ids = []
+    
+    for message in messages:
+        if message['role'] == 'assistant':
+            content = message['content']
+            # Handle both string and list content formats
+            if isinstance(content, str):
+                text = content
+            elif isinstance(content, list) and len(content) > 0 and 'text' in content[0]:
+                text = content[0]['text']
+            else:
+                continue
+                
+            # Try to parse as JSON
+            try:
+                commands = json.loads(text)
+                if not isinstance(commands, list):
+                    commands = [commands]
+                    
+                for cmd in commands:
+                    for key, value in cmd.items():
+                        if key == 'delegate_task' and 'log_id' in value:
+                            log_ids.append(value['log_id'])
+            except (json.JSONDecodeError, TypeError, KeyError):
+                # If not JSON, try regex to find log_ids in delegate_task commands
+                matches = re.findall(r'"delegate_task"\s*:\s*{\s*"log_id"\s*:\s*"([^"]+)"', text)
+                log_ids.extend(matches)
+    
+    return log_ids
+
+def get_cache_dir() -> str:
+    """
+    Get the directory for token count cache files.
+    Creates the directory if it doesn't exist.
+    """
+    cache_dir = os.environ.get('TOKEN_CACHE_DIR', 'data/token_cache')
+    if not os.path.exists(cache_dir):
+        os.makedirs(cache_dir)
+    return cache_dir
+
+def get_cache_path(log_id: str) -> str:
+    """
+    Get the path to the cache file for a specific log_id.
+    """
+    cache_dir = get_cache_dir()
+    return os.path.join(cache_dir, f"tokens_{log_id}.json")
+
+def get_cached_token_counts(log_id: str, log_path: str) -> Dict[str, int]:
+    """
+    Get cached token counts if available and valid.
+    
+    Args:
+        log_id: The log ID
+        log_path: Path to the actual log file
+        
+    Returns:
+        Cached token counts if valid, None otherwise
+    """
+    cache_path = get_cache_path(log_id)
+    
+    # If cache doesn't exist, return None
+    if not os.path.exists(cache_path):
+        return None
+    
+    try:
+        # Get modification times
+        log_mtime = os.path.getmtime(log_path)
+        cache_mtime = os.path.getmtime(cache_path)
+        current_time = time.time()
+        
+        # If log was modified after cache was created, cache is invalid
+        if log_mtime > cache_mtime:
+            return None
+        
+        # Don't recalculate sooner than 3 minutes after last calculation
+        if current_time - cache_mtime < 180:  # 3 minutes in seconds
+            with open(cache_path, 'r') as f:
+                return json.load(f)
+                
+        # For logs that haven't been modified in over an hour, consider them "finished"
+        # and use the cache regardless of when it was last calculated
+        if current_time - log_mtime > 3600:  # 1 hour in seconds
+            with open(cache_path, 'r') as f:
+                return json.load(f)
+    
+    except (json.JSONDecodeError, IOError) as e:
+        print(f"Error reading token cache: {e}")
+    
+    return None
+
+def save_token_counts_to_cache(log_id: str, token_counts: Dict[str, int]) -> None:
+    """
+    Save token counts to cache.
+    """
+    cache_path = get_cache_path(log_id)
+    with open(cache_path, 'w') as f:
+        json.dump(token_counts, f)
+
+def count_tokens_for_log_id(log_id: str) -> Dict[str, int]:
+    """
+    Count tokens for a chat log identified by log_id, including any delegated tasks.
+    
+    Args:
+        log_id: The log ID to count tokens for
+        
+    Returns:
+        Dictionary with token counts or None if log not found
+    """
+    # Find the chatlog file
+    chatlog_path = find_chatlog_file(log_id)
+    if not chatlog_path:
+        return None
+    
+    # Check cache first
+    cached_counts = get_cached_token_counts(log_id, chatlog_path)
+    if cached_counts:
+        print(f"Using cached token counts for {log_id}")
+        return cached_counts
+    
+    print(f"Calculating token counts for {log_id}")
+    
+    # Load the chat log
+    with open(chatlog_path, 'r') as f:
+        log_data = json.load(f)
+        
+    # Create a temporary ChatLog instance to count tokens
+    temp_log = ChatLog(log_id=log_id, user="system", agent=log_data.get('agent', 'unknown'))
+    temp_log.messages = log_data.get('messages', [])
+    
+    # Count tokens for this log
+    parent_counts = temp_log.count_tokens()
+    
+    # Create combined counts (starting with parent counts)
+    combined_counts = {}
+    combined_counts['input_tokens_sequence'] = parent_counts['input_tokens_sequence']
+    combined_counts['output_tokens_sequence'] = parent_counts['output_tokens_sequence']
+    combined_counts['input_tokens_total'] = parent_counts['input_tokens_total']
+    
+    # Find delegated task log IDs
+    delegated_log_ids = extract_delegate_task_log_ids(temp_log.messages)
+    
+    # Recursively count tokens for delegated tasks
+    for delegated_id in delegated_log_ids:
+        delegated_counts = count_tokens_for_log_id(delegated_id)
+        if delegated_counts:
+            combined_counts['input_tokens_sequence'] += delegated_counts['input_tokens_sequence']
+            combined_counts['output_tokens_sequence'] += delegated_counts['output_tokens_sequence']
+            combined_counts['input_tokens_total'] += delegated_counts['input_tokens_total']
+    
+    # Create final result with both parent and combined counts
+    token_counts = {}
+    # Parent session only counts
+    token_counts['input_tokens_sequence'] = parent_counts['input_tokens_sequence']
+    token_counts['output_tokens_sequence'] = parent_counts['output_tokens_sequence']
+    token_counts['input_tokens_total'] = parent_counts['input_tokens_total']
+    # Combined counts (parent + all subtasks)
+    token_counts['combined_input_tokens_sequence'] = combined_counts['input_tokens_sequence']
+    token_counts['combined_output_tokens_sequence'] = combined_counts['output_tokens_sequence']
+    token_counts['combined_input_tokens_total'] = combined_counts['input_tokens_total']
+    
+    # Save to cache
+    save_token_counts_to_cache(log_id, token_counts)
+    
+    return token_counts

{mindroot-8.2.0.dist-info → mindroot-8.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mindroot
-Version: 8.2.0
+Version: 8.4.0
 Summary: MindRoot AI Agent Framework
 Requires-Python: >=3.9
 License-File: LICENSE

{mindroot-8.2.0.dist-info → mindroot-8.4.0.dist-info}/RECORD

@@ -449,8 +449,8 @@ mindroot/coreplugins/chat/commands.py,sha256=QozVxkLqtP6rlK3tSVLThfZz6nE7G_Yygcm
 mindroot/coreplugins/chat/format_result_msgs.py,sha256=daEdpEyAJIa8b2VkCqSKcw8PaExcB6Qro80XNes_sHA,2
 mindroot/coreplugins/chat/mod.py,sha256=Xydjv3feKJJRbwdiB7raqiQnWtaS_2GcdC9bXYQX3nE,425
 mindroot/coreplugins/chat/models.py,sha256=GRcRuDUAJFpyWERPMxkxUaZ21igNlWeeamriruEKiEQ,692
-mindroot/coreplugins/chat/router.py,sha256=Cx6cmdqt5itlcIhBwHmwauR4z5brFdJrCXWSPBVTQHI,7811
-mindroot/coreplugins/chat/services.py,sha256=IiUBjsRzKQZ0xw6TV8-dj6mehzTlA5XxcPPQYFQH6Kk,17561
+mindroot/coreplugins/chat/router.py,sha256=ucSQ6_wztDq2PCUP0D4KHL5JaFUAaBtmCQy-4iI8e8c,9087
+mindroot/coreplugins/chat/services.py,sha256=Ngv_TceFeXn2EzlZbSbR8Iv1E4M7qL7o3q1L7tufFq4,17835
 mindroot/coreplugins/chat/utils.py,sha256=BiE14PpsAcQSO5vbU88klHGm8cAXJDXxgVgva-EXybU,155
 mindroot/coreplugins/chat/static/assistant.png,sha256=oAt1ctkFKLSPBoAZGNnSixooW9ANVIk1GwniauVWDXo,215190
 mindroot/coreplugins/chat/static/mindgen.png,sha256=fN3E3oOFvAGYjJq-Pvg2f75jIMv7kg5WRU0EeEbxCWg,235353
@@ -1765,9 +1765,11 @@ mindroot/coreplugins/user_service/role_service.py,sha256=e6XrxhMC4903C-Y515XSC54
 mindroot/coreplugins/user_service/backup/admin_service.py,sha256=scc59rxlZz4uuVvgjf-9HL2gKi7-uiCdSt6LjWJILR8,4259
 mindroot/coreplugins/user_service/backup/admin_setup.py,sha256=JGszAw8nVtnNiisSUGu9jtoStKGyN44KpbRlKAhDJho,3001
 mindroot/lib/__init__.py,sha256=388n_hMskU0TnZ4xT10US_kFkya-EPBjWcv7AZf_HOk,74
+mindroot/lib/buchatlog.py,sha256=LJZc3ksKgJcStltmHrrwNLaON3EDzhOKVAWj0Wl22wk,5861
+mindroot/lib/buchatlog2.py,sha256=Va9FteBWePEjWD9OZcw-OtQfEb-IoCVGTmJeMRaX9is,13729
 mindroot/lib/butemplates.py,sha256=gfHGPTOjvoEenXsR7xokNuqMjOAPuC2DawheH1Ae4bU,12196
 mindroot/lib/chatcontext.py,sha256=OR63K63NMjaV7kXDf0MIcvOXFZhdiqr7LKgbyfZzjkE,6211
-mindroot/lib/chatlog.py,sha256=B5kLuwMHyEC_AmjNbXWal-tdXwVnOwcZ4PPUK7Vb7NQ,5860
+mindroot/lib/chatlog.py,sha256=6brFJASM7r1qlRX3t1HsXHehEnkaUW5dwmCG3WIPOqc,14690
 mindroot/lib/json_escape.py,sha256=5cAmAdNbnYX2uyfQcnse2fFtNI0CdB-AfZ23RwaDm-k,884
 mindroot/lib/model_selector.py,sha256=Wz-8NZoiclmnhLeCNnI3WCuKFmjsO5HE4bK5F8GpZzU,1397
 mindroot/lib/parent_templates.py,sha256=elcQFFwrFtfAYfQOSTs06aiDDigN1f1R2f8I1V-wj6Q,2731
@@ -1777,6 +1779,7 @@ mindroot/lib/route_decorators.py,sha256=L3E-bn48zhuxk6YPtyc2oP76-5WuV_SmjxtngJeY
 mindroot/lib/session_files.py,sha256=Vl50YfEnKt8ucGSDIavsAdbAcf8IbMG3Fdgm4pEh2Yg,1263
 mindroot/lib/streamcmd.py,sha256=f9n3OtryEkMbNNuFr5BAZn1EpSLUKuDZw-zpo97XxJk,4714
 mindroot/lib/templates.py,sha256=5dODCS6UeC9Y_PdMWlUuQCCZUUt2ICR0S1YF6XrG3eM,15154
+mindroot/lib/token_counter.py,sha256=U3tyw2CG1uK1FmupOHzTkyBwx5UKI30hRrRTqu_-ALQ,7170
 mindroot/lib/auth/__init__.py,sha256=5EZbCTcdlnTHYE0JNk8znWNSO7mOsokMOvRBjb5Mq-M,49
 mindroot/lib/auth/auth.py,sha256=2vhF_LfZcTPt2N2VLWy1ZP7h2pKFv7XM3xW1iRVOTkU,3129
 mindroot/lib/db/organize_models.py,sha256=kiadXfhGjCY16c36l1JmxXcKSH4ShWWEUnHi7WRRn9c,5028
@@ -1814,9 +1817,9 @@ mindroot/protocols/services/stream_chat.py,sha256=fMnPfwaB5fdNMBLTEg8BXKAGvrELKH
 mindroot/registry/__init__.py,sha256=40Xy9bmPHsgdIrOzbtBGzf4XMqXVi9P8oZTJhn0r654,151
 mindroot/registry/component_manager.py,sha256=WZFNPg4SNvpqsM5NFiC2DpgmrJQCyR9cNhrCBpp30Qk,995
 mindroot/registry/data_access.py,sha256=NgNMamxIjaKeYxzxnVaQz1Y-Rm0AI51si3788_JHUTM,5316
-mindroot-8.2.0.dist-info/licenses/LICENSE,sha256=8plAmZh8y9ccuuqFFz4kp7G-cO_qsPgAOoHNvabSB4U,1070
-mindroot-8.2.0.dist-info/METADATA,sha256=aNAkb4BPgjjdy_1OxxFYwxh0GdQPYtrL6pWlfLLotaY,356
-mindroot-8.2.0.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-mindroot-8.2.0.dist-info/entry_points.txt,sha256=0bpyjMccLttx6VcjDp6zfJPN0Kk0rffor6IdIbP0j4c,50
-mindroot-8.2.0.dist-info/top_level.txt,sha256=gwKm7DmNjhdrCJTYCrxa9Szne4lLpCtrEBltfsX-Mm8,9
-mindroot-8.2.0.dist-info/RECORD,,
+mindroot-8.4.0.dist-info/licenses/LICENSE,sha256=8plAmZh8y9ccuuqFFz4kp7G-cO_qsPgAOoHNvabSB4U,1070
+mindroot-8.4.0.dist-info/METADATA,sha256=dL3FgMag3dUJHTDub3nQuv6Q0j2YDo6h8CuDNpaGKdA,356
+mindroot-8.4.0.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+mindroot-8.4.0.dist-info/entry_points.txt,sha256=0bpyjMccLttx6VcjDp6zfJPN0Kk0rffor6IdIbP0j4c,50
+mindroot-8.4.0.dist-info/top_level.txt,sha256=gwKm7DmNjhdrCJTYCrxa9Szne4lLpCtrEBltfsX-Mm8,9
+mindroot-8.4.0.dist-info/RECORD,,

{mindroot-8.2.0.dist-info → mindroot-8.4.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.7.1)
+Generator: setuptools (80.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any