jarvis-ai-assistant 0.1.109__py3-none-any.whl → 0.1.110__py3-none-any.whl

jarvis/__init__.py

@@ -1,3 +1,3 @@
 """Jarvis AI Assistant"""
 
-__version__ = "0.1.109" 
+__version__ = "0.1.110" 

jarvis/agent.py

@@ -8,7 +8,7 @@ import yaml
 from jarvis.jarvis_platform.base import BasePlatform
 from jarvis.jarvis_platform.registry import PlatformRegistry
 from jarvis.jarvis_tools.registry import ToolRegistry, tool_call_help
-from jarvis.utils import PrettyOutput, OutputType, get_context_token_count, is_auto_complete, is_need_summary, is_record_methodology, load_methodology, add_agent, delete_current_agent, get_max_context_length, get_multiline_input, init_env, is_use_methodology
+from jarvis.utils import PrettyOutput, OutputType, get_context_token_count, is_auto_complete, is_need_summary, is_record_methodology, load_methodology, add_agent, delete_current_agent, get_max_token_count, get_multiline_input, init_env, is_use_methodology
 import os
 
 class Agent:
@@ -89,7 +89,7 @@ class Agent:
 Please describe in concise bullet points, highlighting important information.
 """
         
-        self.max_context_length = max_context_length if max_context_length is not None else get_max_context_length()
+        self.max_token_count = max_context_length if max_context_length is not None else get_max_token_count()
 
         self.auto_complete = auto_complete if auto_complete is not None else is_auto_complete()
 
@@ -325,7 +325,7 @@ Please continue the task based on the above information.
                     self.conversation_length += get_context_token_count(self.prompt)
                     
                     # 如果对话历史长度超过限制，在提示中添加提醒
-                    if self.conversation_length > self.max_context_length:
+                    if self.conversation_length > self.max_token_count:
                         current_response = self._summarize_and_clear_history()
                         continue
                     else:

jarvis/jarvis_codebase/main.py

@@ -7,7 +7,7 @@ from typing import List, Tuple, Optional, Dict
 from jarvis.jarvis_platform.registry import PlatformRegistry
 import concurrent.futures
 from concurrent.futures import ThreadPoolExecutor
-from jarvis.utils import OutputType, PrettyOutput, find_git_root, get_context_token_count, get_embedding, get_file_md5, get_max_context_length, get_thread_count, load_embedding_model, user_confirm
+from jarvis.utils import OutputType, PrettyOutput, find_git_root, get_context_token_count, get_embedding, get_file_md5, get_max_token_count, get_thread_count, load_embedding_model, user_confirm
 from jarvis.utils import init_env
 import argparse
 import pickle
@@ -21,7 +21,7 @@ class CodeBase:
         self.root_dir = root_dir
         os.chdir(self.root_dir)
         self.thread_count = get_thread_count()
-        self.max_context_length = get_max_context_length()
+        self.max_token_count = get_max_token_count()
         self.index = None
             
         # 初始化数据目录
@@ -218,7 +218,7 @@ Code content:
                 return cached_vector
                 
             # Read the file content and combine information
-            content = open(file_path, "r", encoding="utf-8").read()[:self.max_context_length]  # Limit the file content length
+            content = open(file_path, "r", encoding="utf-8").read()[:self.max_token_count]  # Limit the file content length
             
             # Combine file information, including file content
             combined_text = f"""
@@ -541,7 +541,7 @@ Content: {content}
             PrettyOutput.print(f"Picking results for query: \n" + "\n".join(query), output_type=OutputType.INFO)
             
             # Maximum content length per batch
-            max_batch_length = self.max_context_length - 1000  # Reserve space for prompt
+            max_batch_length = self.max_token_count - 1000  # Reserve space for prompt
             max_file_length = max_batch_length // 3  # Limit individual file size
             
             # Process files in batches
@@ -805,7 +805,7 @@ Question: {query}
 Relevant code files (ordered by relevance):
 """
         # Add context with length control
-        available_count = self.max_context_length - get_context_token_count(prompt) - 1000  # Reserve space for answer
+        available_count = self.max_token_count - get_context_token_count(prompt) - 1000  # Reserve space for answer
         current_count = 0
         
         for path in files_from_codebase:

jarvis/jarvis_platform/oyi.py

@@ -2,7 +2,7 @@ import mimetypes
 import os
 from typing import Dict, List, Tuple
 from jarvis.jarvis_platform.base import BasePlatform
-from jarvis.utils import PrettyOutput, OutputType, get_max_context_length
+from jarvis.utils import PrettyOutput, OutputType
 import requests
 import json
 

jarvis/jarvis_rag/main.py

@@ -3,7 +3,7 @@ import numpy as np
 import faiss
 from typing import List, Tuple, Optional, Dict
 import pickle
-from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_embedding, get_embedding_batch, get_file_md5, get_max_context_length, get_max_paragraph_length, get_min_paragraph_length, get_thread_count, init_gpu_config, load_embedding_model
+from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_embedding, get_embedding_batch, get_file_md5, get_max_token_count, get_max_paragraph_length, get_min_paragraph_length, get_thread_count, init_gpu_config, load_embedding_model
 from jarvis.utils import init_env
 from dataclasses import dataclass
 from tqdm import tqdm
@@ -143,7 +143,7 @@ class RAGTool:
         self.min_paragraph_length = get_min_paragraph_length()  # Minimum paragraph length
         self.max_paragraph_length = get_max_paragraph_length()  # Maximum paragraph length
         self.context_window = 5  # Fixed context window size
-        self.max_context_length = int(get_max_context_length() * 0.8)
+        self.max_token_count = int(get_max_token_count() * 0.8)
         
         # Initialize data directory
         self.data_dir = os.path.join(self.root_dir, ".jarvis/rag")
@@ -655,7 +655,7 @@ Question: {question}
 Relevant documents (ordered by relevance):
 """
             # 添加上下文，控制长度
-            available_count = self.max_context_length - get_context_token_count(prompt) - 1000
+            available_count = self.max_token_count - get_context_token_count(prompt) - 1000
             current_count = 0
             
             for doc, score in results:

jarvis/jarvis_tools/registry.py

@@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Optional
 
 from jarvis.jarvis_platform.registry import PlatformRegistry
 from jarvis.jarvis_tools.base import Tool
-from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_max_context_length
+from jarvis.utils import OutputType, PrettyOutput, get_context_token_count, get_max_token_count
 
 
 tool_call_help = """## Tool Usage Format
@@ -64,8 +64,8 @@ class ToolRegistry:
         # Load built-in tools and external tools
         self._load_builtin_tools()
         self._load_external_tools()
-        # Ensure max_context_length is an integer
-        self.max_context_length = int(get_max_context_length() * 0.8)
+        # Ensure max_token_count is an integer
+        self.max_token_count = int(get_max_token_count() * 0.8)
 
     def use_tools(self, name: List[str]):
         """Use specified tools"""
@@ -248,13 +248,13 @@ arguments:
                 PrettyOutput.section("Execution successful", OutputType.SUCCESS)
                 
                 # If the output exceeds 4k characters, use a large model to summarize
-                if get_context_token_count(output) > self.max_context_length:
+                if get_context_token_count(output) > self.max_token_count:
                     try:
                         PrettyOutput.print("Output is too long, summarizing...", OutputType.PROGRESS)
                         model = PlatformRegistry.get_global_platform_registry().get_normal_platform()
                         
                         # If the output exceeds the maximum context length, only take the last part
-                        max_count = self.max_context_length
+                        max_count = self.max_token_count
                         if get_context_token_count(output) > max_count:
                             output_to_summarize = output[-max_count:]
                             truncation_notice = f"\n(Note: Due to the length of the output, only the last {max_count} characters are summarized)"

jarvis/utils.py

@@ -463,8 +463,8 @@ def load_rerank_model():
 
 def is_long_context(files: list) -> bool:
     """Check if the file list belongs to a long context (total characters exceed 80% of the maximum context length)"""
-    max_length = get_max_context_length()
-    threshold = max_length * 0.8
+    max_token_count = get_max_token_count()
+    threshold = max_token_count * 0.8
     total_tokens = 0
     
     for file_path in files:
@@ -674,8 +674,8 @@ def get_embedding_batch(embedding_model: Any, texts: List[str]) -> np.ndarray:
 
 
     
-def get_max_context_length():
-    return int(os.getenv('JARVIS_MAX_CONTEXT_LENGTH', '131072'))  # 默认128k
+def get_max_token_count():
+    return int(os.getenv('JARVIS_MAX_TOKEN_COUNT', '131072'))  # 默认128k
     
 def get_thread_count():
     return int(os.getenv('JARVIS_THREAD_COUNT', '1'))  
@@ -774,8 +774,8 @@ def get_context_token_count(text: str) -> int:
     try:
         # Use a fast tokenizer that's good at general text
         tokenizer = load_tokenizer()
-        tokens = tokenizer.encode(text)
-        return len(tokens)
+        chunks = split_text_into_chunks(text, 512)
+        return sum([len(tokenizer.encode(chunk)) for chunk in chunks])
         
     except Exception as e:
         PrettyOutput.print(f"Error counting tokens: {str(e)}", OutputType.WARNING)

{jarvis_ai_assistant-0.1.109.dist-info → jarvis_ai_assistant-0.1.110.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: jarvis-ai-assistant
-Version: 0.1.109
+Version: 0.1.110
 Summary: Jarvis: An AI assistant that uses tools to interact with the system
 Home-page: https://github.com/skyfireitdiy/Jarvis
 Author: skyfire

{jarvis_ai_assistant-0.1.109.dist-info → jarvis_ai_assistant-0.1.110.dist-info}/RECORD

@@ -1,13 +1,13 @@
-jarvis/__init__.py,sha256=CB3MYpgrHjKZT-LJIoysQYNgLyXYOJSKmxopMqJsAmU,51
-jarvis/agent.py,sha256=x6LmdW5k50nhAK6hdb8V1Vvkt64szwrnaewAto8murg,22690
-jarvis/utils.py,sha256=_GQY1UlxzhKM0DwdnuTUeSmaae8dV83SffsjMPPXFM0,26889
+jarvis/__init__.py,sha256=0rw1RD8VrgfjPxXtqbnGEx3Aj9K4mgrYWuVSzACO3Ic,51
+jarvis/agent.py,sha256=eV2Bgm5Q6gnQb2QeEo9bHCDaLY0v3RSV8Ylm_gS2_Yc,22678
+jarvis/utils.py,sha256=7v9hs9Tlyi9XMLYkPUzbzzMXGuAlJAx5999O2d7kP9A,26945
 jarvis/jarvis_code_agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jarvis/jarvis_code_agent/code_agent.py,sha256=nigsmCK6D2z0dFU_1HFNYEvXr3lWdl0rm6p4VgiOk6o,5980
 jarvis/jarvis_code_agent/file_select.py,sha256=1kOVRLPS1GZcDyGpCW9hOPbfCEwF8f0-qVPaRZPHzoM,8154
 jarvis/jarvis_code_agent/patch.py,sha256=bOhegGKs4JEmJJOZfUlmwzGI6kakMyi2Q62HADJ7Npk,4594
 jarvis/jarvis_code_agent/relevant_files.py,sha256=Q4nI45zuyWt5aKuc4OR7-a6UbOXOym3oEzQJvqxkF8Q,946
 jarvis/jarvis_codebase/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jarvis/jarvis_codebase/main.py,sha256=7SwL0QPeojsC95SIlbKjq9FcvKnvPBbtH1kv3SjPUBY,39701
+jarvis/jarvis_codebase/main.py,sha256=40ySTIrQld5uW60vW9pawZopjDNVvtmlW27oNVaJXH0,39683
 jarvis/jarvis_lsp/base.py,sha256=_7pdbMKjdtYBW0DsRbjIodDHM3J7df-YgXHejN_WIrU,4490
 jarvis/jarvis_lsp/cpp.py,sha256=F7Zo3BErkvtWS1_H9zQO83pX_FUmnijux-2SjhWzKCE,4985
 jarvis/jarvis_lsp/go.py,sha256=p8LULiFdq4qjDYQzXFlzH0-FQZ3IyfiwN_sbO9i0L_A,5310
@@ -20,12 +20,12 @@ jarvis/jarvis_platform/base.py,sha256=nQ-rsJL1Z-gMev3TPoY7tYdwxhCJY8LG6_gtJ-maiW
 jarvis/jarvis_platform/kimi.py,sha256=3yiOL2PsEcKEL0Yj0Hm3lTg9M0Ahy0Ou1AUnJ0AS0Ss,15768
 jarvis/jarvis_platform/ollama.py,sha256=9Ptu-UzRMnNxqFlx9uDpHO0_Imrzf0Wfw9sZqnv2wRI,5681
 jarvis/jarvis_platform/openai.py,sha256=NYAIaQbFH9Usg5ZxkBSek1F0imu-pDB9Qf6Am0AtU0s,4130
-jarvis/jarvis_platform/oyi.py,sha256=mV8tsQty2Htz--DNemBAnCiauih3JQ4jSyuZi5L4WQo,15089
+jarvis/jarvis_platform/oyi.py,sha256=11WcpJu0rsQfcHP1SVVwpbOjBoJzvQ6LELHWQMCvyzw,15065
 jarvis/jarvis_platform/registry.py,sha256=9QLoihcnkYckrCzgNnlTqaLn_z_HMhaxMSyUNb8IEys,8538
 jarvis/jarvis_platform_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jarvis/jarvis_platform_manager/main.py,sha256=17607aNAStqJ1sOQLTGi6Tnv-cIQme_r5YvbB_S3enc,4985
 jarvis/jarvis_rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-jarvis/jarvis_rag/main.py,sha256=ZdZZnOyQX8q1mNUeMymb9qg1l77JnG6WBz_KDov0NLQ,31366
+jarvis/jarvis_rag/main.py,sha256=Lr3b2eTB9TXZGZGdG4Sl9bdtE5NFRbv_bRysxeWNCEo,31354
 jarvis/jarvis_smart_shell/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 jarvis/jarvis_smart_shell/main.py,sha256=VdUR-x932OccEwU0pcQM_pb_I4yfrAutE3hfm6jf5es,3955
 jarvis/jarvis_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -51,12 +51,12 @@ jarvis/jarvis_tools/methodology.py,sha256=RFqcVjKuj8ESGmNYcQz_HyphsitDvF3XtqgGaq
 jarvis/jarvis_tools/rag.py,sha256=2fQHqc4bw8JM-OxGTsHobLIOTo8Mip3rdtJCmAoY8XU,4952
 jarvis/jarvis_tools/read_code.py,sha256=5DGmeXTgumAiG0RP1xB4sF4NdmBm5BEGjRRlIBzjGnQ,4002
 jarvis/jarvis_tools/read_webpage.py,sha256=JCReSXhkDHDkQ606sZYIKG1Itlprjpmu1sSbF-Ed-jI,2478
-jarvis/jarvis_tools/registry.py,sha256=mYx7q78BQ9El68c6DPbbeMyRp0Dr9GnZoozjhSViLPo,11859
+jarvis/jarvis_tools/registry.py,sha256=OR-BxSVfI3ER_1rAPMZfLf45E2YpheeS01j8MJ8RGso,11841
 jarvis/jarvis_tools/search.py,sha256=PLSSNETyajpqDoStCTfkoy-D41IMNudTuVzonMlT6Aw,9225
 jarvis/jarvis_tools/select_code_files.py,sha256=bjJGwCNw0Ue_8jW60K1gcy1rUgKqoHihicu5SS58WNk,1890
-jarvis_ai_assistant-0.1.109.dist-info/LICENSE,sha256=AGgVgQmTqFvaztRtCAXsAMryUymB18gZif7_l2e1XOg,1063
-jarvis_ai_assistant-0.1.109.dist-info/METADATA,sha256=pxorU-ZxDXj6dNQOZfIh285QvRa9hUxXmKGVbgotBTc,14392
-jarvis_ai_assistant-0.1.109.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-jarvis_ai_assistant-0.1.109.dist-info/entry_points.txt,sha256=UYj4FYvOH8jJ0GgCJTA_TAmJ3wvikos-hUVbCwt_KOc,480
-jarvis_ai_assistant-0.1.109.dist-info/top_level.txt,sha256=1BOxyWfzOP_ZXj8rVTDnNCJ92bBGB0rwq8N1PCpoMIs,7
-jarvis_ai_assistant-0.1.109.dist-info/RECORD,,
+jarvis_ai_assistant-0.1.110.dist-info/LICENSE,sha256=AGgVgQmTqFvaztRtCAXsAMryUymB18gZif7_l2e1XOg,1063
+jarvis_ai_assistant-0.1.110.dist-info/METADATA,sha256=0Ogu7gge_EAhRT6p7HXEqeGNb4uAFMa6MedSJQsp0yY,14392
+jarvis_ai_assistant-0.1.110.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+jarvis_ai_assistant-0.1.110.dist-info/entry_points.txt,sha256=UYj4FYvOH8jJ0GgCJTA_TAmJ3wvikos-hUVbCwt_KOc,480
+jarvis_ai_assistant-0.1.110.dist-info/top_level.txt,sha256=1BOxyWfzOP_ZXj8rVTDnNCJ92bBGB0rwq8N1PCpoMIs,7
+jarvis_ai_assistant-0.1.110.dist-info/RECORD,,