jarvis-ai-assistant 0.1.109__tar.gz → 0.1.110__tar.gz

{jarvis_ai_assistant-0.1.109/src/jarvis_ai_assistant.egg-info → jarvis_ai_assistant-0.1.110}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: jarvis-ai-assistant
-Version: 0.1.109
+Version: 0.1.110
 Summary: Jarvis: An AI assistant that uses tools to interact with the system
 Home-page: https://github.com/skyfireitdiy/Jarvis
 Author: skyfire

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jarvis-ai-assistant"
-version = "0.1.109"
+version = "0.1.110"
 description = "Jarvis: An AI assistant that uses tools to interact with the system"
 readme = "README.md"
 authors = [{ name = "Your Name", email = "your.email@example.com" }]

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="jarvis-ai-assistant",
-    version="0.1.109",
+    version="0.1.110",
     author="skyfire",
     author_email="skyfireitdiy@hotmail.com",
     description="An AI assistant that uses various tools to interact with the system",

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/__init__.py

@@ -1,3 +1,3 @@
 """Jarvis AI Assistant"""
 
-__version__ = "0.1.109" 
+__version__ = "0.1.110" 

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/agent.py

@@ -8,7 +8,7 @@ import yaml
 from jarvis.jarvis_platform.base import BasePlatform
 from jarvis.jarvis_platform.registry import PlatformRegistry
 from jarvis.jarvis_tools.registry import ToolRegistry, tool_call_help
-from jarvis.utils import PrettyOutput, OutputType, get_context_token_count, is_auto_complete, is_need_summary, is_record_methodology, load_methodology, add_agent, delete_current_agent, get_max_context_length, get_multiline_input, init_env, is_use_methodology
+from jarvis.utils import PrettyOutput, OutputType, get_context_token_count, is_auto_complete, is_need_summary, is_record_methodology, load_methodology, add_agent, delete_current_agent, get_max_token_count, get_multiline_input, init_env, is_use_methodology
 import os
 
 class Agent:
@@ -89,7 +89,7 @@ class Agent:
 Please describe in concise bullet points, highlighting important information.
 """
         
-        self.max_context_length = max_context_length if max_context_length is not None else get_max_context_length()
+        self.max_token_count = max_context_length if max_context_length is not None else get_max_token_count()
 
         self.auto_complete = auto_complete if auto_complete is not None else is_auto_complete()
 
@@ -325,7 +325,7 @@ Please continue the task based on the above information.
                     self.conversation_length += get_context_token_count(self.prompt)
                     
                     # 如果对话历史长度超过限制，在提示中添加提醒
-                    if self.conversation_length > self.max_context_length:
+                    if self.conversation_length > self.max_token_count:
                         current_response = self._summarize_and_clear_history()
                         continue
                     else:

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/jarvis_codebase/main.py

@@ -7,7 +7,7 @@ from typing import List, Tuple, Optional, Dict
 from jarvis.jarvis_platform.registry import PlatformRegistry
 import concurrent.futures
 from concurrent.futures import ThreadPoolExecutor
-from jarvis.utils import OutputType, PrettyOutput, find_git_root, get_context_token_count, get_embedding, get_file_md5, get_max_context_length, get_thread_count, load_embedding_model, user_confirm
+from jarvis.utils import OutputType, PrettyOutput, find_git_root, get_context_token_count, get_embedding, get_file_md5, get_max_token_count, get_thread_count, load_embedding_model, user_confirm
 from jarvis.utils import init_env
 import argparse
 import pickle
@@ -21,7 +21,7 @@ class CodeBase:
         self.root_dir = root_dir
         os.chdir(self.root_dir)
         self.thread_count = get_thread_count()
-        self.max_context_length = get_max_context_length()
+        self.max_token_count = get_max_token_count()
         self.index = None
             
         # 初始化数据目录
@@ -218,7 +218,7 @@ Code content:
                 return cached_vector
                 
             # Read the file content and combine information
-            content = open(file_path, "r", encoding="utf-8").read()[:self.max_context_length]  # Limit the file content length
+            content = open(file_path, "r", encoding="utf-8").read()[:self.max_token_count]  # Limit the file content length
             
             # Combine file information, including file content
             combined_text = f"""
@@ -541,7 +541,7 @@ Content: {content}
             PrettyOutput.print(f"Picking results for query: \n" + "\n".join(query), output_type=OutputType.INFO)
             
             # Maximum content length per batch
-            max_batch_length = self.max_context_length - 1000  # Reserve space for prompt
+            max_batch_length = self.max_token_count - 1000  # Reserve space for prompt
             max_file_length = max_batch_length // 3  # Limit individual file size
             
             # Process files in batches
@@ -805,7 +805,7 @@ Question: {query}
 Relevant code files (ordered by relevance):
 """
         # Add context with length control
-        available_count = self.max_context_length - get_context_token_count(prompt) - 1000  # Reserve space for answer
+        available_count = self.max_token_count - get_context_token_count(prompt) - 1000  # Reserve space for answer
         current_count = 0
         
         for path in files_from_codebase:

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/jarvis_platform/oyi.py

@@ -2,7 +2,7 @@ import mimetypes
 import os
 from typing import Dict, List, Tuple
 from jarvis.jarvis_platform.base import BasePlatform
-from jarvis.utils import PrettyOutput, OutputType, get_max_context_length
+from jarvis.utils import PrettyOutput, OutputType
 import requests
 import json
 

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/jarvis_rag/main.py

@@ -3,7 +3,7 @@ import numpy as np
 import faiss
 from typing import List, Tuple, Optional, Dict
 import pickle
-from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_embedding, get_embedding_batch, get_file_md5, get_max_context_length, get_max_paragraph_length, get_min_paragraph_length, get_thread_count, init_gpu_config, load_embedding_model
+from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_embedding, get_embedding_batch, get_file_md5, get_max_token_count, get_max_paragraph_length, get_min_paragraph_length, get_thread_count, init_gpu_config, load_embedding_model
 from jarvis.utils import init_env
 from dataclasses import dataclass
 from tqdm import tqdm
@@ -143,7 +143,7 @@ class RAGTool:
         self.min_paragraph_length = get_min_paragraph_length()  # Minimum paragraph length
         self.max_paragraph_length = get_max_paragraph_length()  # Maximum paragraph length
         self.context_window = 5  # Fixed context window size
-        self.max_context_length = int(get_max_context_length() * 0.8)
+        self.max_token_count = int(get_max_token_count() * 0.8)
         
         # Initialize data directory
         self.data_dir = os.path.join(self.root_dir, ".jarvis/rag")
@@ -655,7 +655,7 @@ Question: {question}
 Relevant documents (ordered by relevance):
 """
             # 添加上下文，控制长度
-            available_count = self.max_context_length - get_context_token_count(prompt) - 1000
+            available_count = self.max_token_count - get_context_token_count(prompt) - 1000
             current_count = 0
             
             for doc, score in results:

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/jarvis_tools/registry.py

@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Optional
 
 from jarvis.jarvis_platform.registry import PlatformRegistry
 from jarvis.jarvis_tools.base import Tool
-from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_max_context_length
+from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_max_token_count
 
 
 tool_call_help = """## Tool Usage Format
@@ -64,8 +64,8 @@ class ToolRegistry:
         # Load built-in tools and external tools
         self._load_builtin_tools()
         self._load_external_tools()
-        # Ensure max_context_length is an integer
-        self.max_context_length = int(get_max_context_length() * 0.8)
+        # Ensure max_token_count is an integer
+        self.max_token_count = int(get_max_token_count() * 0.8)
 
     def use_tools(self, name: List[str]):
         """Use specified tools"""
@@ -248,13 +248,13 @@ arguments:
                 PrettyOutput.section("Execution successful", OutputType.SUCCESS)
                 
                 # If the output exceeds 4k characters, use a large model to summarize
-                if get_context_token_count(output) > self.max_context_length:
+                if get_context_token_count(output) > self.max_token_count:
                     try:
                         PrettyOutput.print("Output is too long, summarizing...", OutputType.PROGRESS)
                         model = PlatformRegistry.get_global_platform_registry().get_normal_platform()
                         
                         # If the output exceeds the maximum context length, only take the last part
-                        max_count = self.max_context_length
+                        max_count = self.max_token_count
                         if get_context_token_count(output) > max_count:
                             output_to_summarize = output[-max_count:]
                             truncation_notice = f"\n(Note: Due to the length of the output, only the last {max_count} characters are summarized)"

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110}/src/jarvis/utils.py

@@ -463,8 +463,8 @@ def load_rerank_model():
 
 def is_long_context(files: list) -> bool:
     """Check if the file list belongs to a long context (total characters exceed 80% of the maximum context length)"""
-    max_length = get_max_context_length()
-    threshold = max_length * 0.8
+    max_token_count = get_max_token_count()
+    threshold = max_token_count * 0.8
     total_tokens = 0
     
     for file_path in files:
@@ -674,8 +674,8 @@ def get_embedding_batch(embedding_model: Any, texts: List[str]) -> np.ndarray:
 
 
     
-def get_max_context_length():
-    return int(os.getenv('JARVIS_MAX_CONTEXT_LENGTH', '131072'))  # 默认128k
+def get_max_token_count():
+    return int(os.getenv('JARVIS_MAX_TOKEN_COUNT', '131072'))  # 默认128k
     
 def get_thread_count():
     return int(os.getenv('JARVIS_THREAD_COUNT', '1'))  
@@ -774,8 +774,8 @@ def get_context_token_count(text: str) -> int:
     try:
         # Use a fast tokenizer that's good at general text
         tokenizer = load_tokenizer()
-        tokens = tokenizer.encode(text)
-        return len(tokens)
+        chunks = split_text_into_chunks(text, 512)
+        return sum([len(tokenizer.encode(chunk)) for chunk in chunks])
         
     except Exception as e:
         PrettyOutput.print(f"Error counting tokens: {str(e)}", OutputType.WARNING)

{jarvis_ai_assistant-0.1.109 → jarvis_ai_assistant-0.1.110/src/jarvis_ai_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: jarvis-ai-assistant
-Version: 0.1.109
+Version: 0.1.110
 Summary: Jarvis: An AI assistant that uses tools to interact with the system
 Home-page: https://github.com/skyfireitdiy/Jarvis
 Author: skyfire