jarvis-ai-assistant 0.1.207__py3-none-any.whl → 0.1.209__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. jarvis/__init__.py +1 -1
  2. jarvis/jarvis_agent/__init__.py +63 -103
  3. jarvis/jarvis_agent/edit_file_handler.py +43 -47
  4. jarvis/jarvis_agent/jarvis.py +33 -39
  5. jarvis/jarvis_code_agent/code_agent.py +74 -30
  6. jarvis/jarvis_code_agent/lint.py +6 -6
  7. jarvis/jarvis_code_analysis/code_review.py +164 -175
  8. jarvis/jarvis_data/config_schema.json +0 -25
  9. jarvis/jarvis_git_utils/git_commiter.py +148 -153
  10. jarvis/jarvis_methodology/main.py +70 -81
  11. jarvis/jarvis_platform/base.py +21 -17
  12. jarvis/jarvis_platform/kimi.py +59 -64
  13. jarvis/jarvis_platform/tongyi.py +118 -131
  14. jarvis/jarvis_platform/yuanbao.py +117 -122
  15. jarvis/jarvis_platform_manager/main.py +102 -502
  16. jarvis/jarvis_platform_manager/service.py +432 -0
  17. jarvis/jarvis_smart_shell/main.py +99 -33
  18. jarvis/jarvis_tools/ask_user.py +0 -1
  19. jarvis/jarvis_tools/edit_file.py +64 -55
  20. jarvis/jarvis_tools/file_analyzer.py +17 -28
  21. jarvis/jarvis_tools/read_code.py +80 -81
  22. jarvis/jarvis_utils/builtin_replace_map.py +1 -36
  23. jarvis/jarvis_utils/config.py +13 -48
  24. jarvis/jarvis_utils/embedding.py +6 -51
  25. jarvis/jarvis_utils/git_utils.py +93 -43
  26. jarvis/jarvis_utils/http.py +104 -0
  27. jarvis/jarvis_utils/methodology.py +12 -17
  28. jarvis/jarvis_utils/utils.py +186 -63
  29. {jarvis_ai_assistant-0.1.207.dist-info → jarvis_ai_assistant-0.1.209.dist-info}/METADATA +4 -19
  30. {jarvis_ai_assistant-0.1.207.dist-info → jarvis_ai_assistant-0.1.209.dist-info}/RECORD +34 -40
  31. {jarvis_ai_assistant-0.1.207.dist-info → jarvis_ai_assistant-0.1.209.dist-info}/entry_points.txt +1 -1
  32. jarvis/jarvis_data/huggingface.tar.gz +0 -0
  33. jarvis/jarvis_dev/main.py +0 -1247
  34. jarvis/jarvis_tools/chdir.py +0 -72
  35. jarvis/jarvis_tools/code_plan.py +0 -218
  36. jarvis/jarvis_tools/create_code_agent.py +0 -95
  37. jarvis/jarvis_tools/create_sub_agent.py +0 -82
  38. jarvis/jarvis_tools/file_operation.py +0 -238
  39. jarvis/jarvis_utils/jarvis_history.py +0 -98
  40. {jarvis_ai_assistant-0.1.207.dist-info → jarvis_ai_assistant-0.1.209.dist-info}/WHEEL +0 -0
  41. {jarvis_ai_assistant-0.1.207.dist-info → jarvis_ai_assistant-0.1.209.dist-info}/licenses/LICENSE +0 -0
  42. {jarvis_ai_assistant-0.1.207.dist-info → jarvis_ai_assistant-0.1.209.dist-info}/top_level.txt +0 -0
@@ -19,8 +19,6 @@
19
19
  """
20
20
  from typing import Any, Dict
21
21
 
22
- from yaspin import yaspin
23
-
24
22
  from jarvis.jarvis_agent.edit_file_handler import EditFileHandler
25
23
 
26
24
 
@@ -62,12 +60,18 @@ class FileSearchReplaceTool:
62
60
  "items": {
63
61
  "type": "object",
64
62
  "properties": {
65
- "reason": {"type": "string", "description": "修改的原因"},
63
+ "reason": {
64
+ "type": "string",
65
+ "description": "修改的原因",
66
+ },
66
67
  "search": {
67
68
  "type": "string",
68
69
  "description": "需要查找的原始代码",
69
70
  },
70
- "replace": {"type": "string", "description": "替换后的新代码"},
71
+ "replace": {
72
+ "type": "string",
73
+ "description": "替换后的新代码",
74
+ },
71
75
  },
72
76
  },
73
77
  },
@@ -79,7 +83,6 @@ class FileSearchReplaceTool:
79
83
  "required": ["files"],
80
84
  }
81
85
 
82
-
83
86
  def execute(self, args: Dict) -> Dict[str, Any]:
84
87
  """执行文件编辑操作,支持快速编辑和AI辅助编辑两种模式。
85
88
 
@@ -152,46 +155,46 @@ class FileSearchReplaceTool:
152
155
  if file_exists and agent:
153
156
  files = agent.get_user_data("files")
154
157
  if not files or file_path not in files:
155
- file_results.append({
156
- "file": file_path,
157
- "success": False,
158
- "stdout": "",
159
- "stderr": f"请先读取文件 {file_path} 的内容后再编辑"
160
- })
158
+ file_results.append(
159
+ {
160
+ "file": file_path,
161
+ "success": False,
162
+ "stdout": "",
163
+ "stderr": f"请先读取文件 {file_path} 的内容后再编辑",
164
+ }
165
+ )
161
166
  continue
162
167
 
163
- with yaspin(
164
- text=f"正在处理文件 {file_path}...", color="cyan"
165
- ) as spinner:
166
- # 首先尝试fast_edit模式
167
- success, temp_content = EditFileHandler._fast_edit(file_path, changes, spinner)
168
+ print(f"⚙️ 正在处理文件 {file_path}...")
169
+ # 首先尝试fast_edit模式
170
+ success, temp_content = EditFileHandler._fast_edit(
171
+ file_path, changes
172
+ )
173
+ if not success:
174
+ # 如果fast_edit失败,尝试slow_edit模式
175
+ success, temp_content = EditFileHandler._slow_edit(
176
+ file_path, changes, agent
177
+ )
168
178
  if not success:
169
- # 如果fast_edit失败,尝试slow_edit模式
170
- success, temp_content = EditFileHandler._slow_edit(
171
- file_path,
172
- changes,
173
- spinner,
174
- agent
175
- )
176
- if not success:
177
- spinner.text = f"文件 {file_path} 处理失败"
178
- spinner.fail("❌")
179
- file_results.append({
179
+ print(f"❌ 文件 {file_path} 处理失败")
180
+ file_results.append(
181
+ {
180
182
  "file": file_path,
181
183
  "success": False,
182
184
  "stdout": "",
183
- "stderr": temp_content
184
- })
185
- continue
186
- else:
187
- spinner.text = f"文件 {file_path} 内容生成完成"
188
- spinner.ok("✅")
185
+ "stderr": temp_content,
186
+ }
187
+ )
188
+ continue
189
189
  else:
190
- spinner.text = f"文件 {file_path} 内容生成完成"
191
- spinner.ok("✅")
190
+ print(f"文件 {file_path} 内容生成完成")
191
+ else:
192
+ print(f"✅ 文件 {file_path} 内容生成完成")
192
193
 
193
194
  # 只有当所有替换操作都成功时,才写回文件
194
- if success and (temp_content != original_content or not file_exists):
195
+ if success and (
196
+ temp_content != original_content or not file_exists
197
+ ):
195
198
  # 确保目录存在
196
199
  os.makedirs(
197
200
  os.path.dirname(os.path.abspath(file_path)), exist_ok=True
@@ -208,24 +211,28 @@ class FileSearchReplaceTool:
208
211
  PrettyOutput.print(stdout_message, OutputType.SUCCESS)
209
212
  overall_success = True
210
213
 
211
- file_results.append({
212
- "file": file_path,
213
- "success": True,
214
- "stdout": stdout_message,
215
- "stderr": ""
216
- })
214
+ file_results.append(
215
+ {
216
+ "file": file_path,
217
+ "success": True,
218
+ "stdout": stdout_message,
219
+ "stderr": "",
220
+ }
221
+ )
217
222
 
218
223
  except Exception as e:
219
224
  stderr_message = f"处理文件 {file_path} 时出错: {str(e)}"
220
225
  stderr_messages.append(stderr_message)
221
226
  PrettyOutput.print(stderr_message, OutputType.WARNING)
222
227
  file_success = False
223
- file_results.append({
224
- "file": file_path,
225
- "success": False,
226
- "stdout": "",
227
- "stderr": stderr_message
228
- })
228
+ file_results.append(
229
+ {
230
+ "file": file_path,
231
+ "success": False,
232
+ "stdout": "",
233
+ "stderr": stderr_message,
234
+ }
235
+ )
229
236
 
230
237
  except Exception as e:
231
238
  error_msg = f"文件搜索替换操作失败: {str(e)}"
@@ -251,12 +258,14 @@ class FileSearchReplaceTool:
251
258
  except:
252
259
  stderr_messages.append(f"回滚文件失败: {file_path}")
253
260
 
254
- file_results.append({
255
- "file": file_path,
256
- "success": False,
257
- "stdout": "",
258
- "stderr": error_msg
259
- })
261
+ file_results.append(
262
+ {
263
+ "file": file_path,
264
+ "success": False,
265
+ "stdout": "",
266
+ "stderr": error_msg,
267
+ }
268
+ )
260
269
 
261
270
  # 整合所有错误信息到stderr
262
271
  all_stderr = []
@@ -267,5 +276,5 @@ class FileSearchReplaceTool:
267
276
  return {
268
277
  "success": overall_success,
269
278
  "stdout": "\n".join(stdout_messages) if overall_success else "",
270
- "stderr": "\n".join(all_stderr) if not overall_success else ""
279
+ "stderr": "\n".join(all_stderr) if not overall_success else "",
271
280
  }
@@ -2,8 +2,6 @@
2
2
  import os
3
3
  from typing import Any, Dict
4
4
 
5
- from yaspin import yaspin # type: ignore
6
- from yaspin.spinners import Spinners # type: ignore
7
5
 
8
6
  from jarvis.jarvis_platform.registry import PlatformRegistry
9
7
  from jarvis.jarvis_utils.output import OutputType, PrettyOutput
@@ -47,9 +45,6 @@ class FileAnalyzerTool:
47
45
  file_paths = args["file_paths"]
48
46
  prompt = args["prompt"]
49
47
 
50
- agent = args["agent"]
51
- agent.reset_tool_call_count()
52
-
53
48
  # 验证文件路径
54
49
  valid_files = []
55
50
  for file_path in file_paths:
@@ -78,28 +73,24 @@ class FileAnalyzerTool:
78
73
  platform.set_system_prompt(system_message)
79
74
 
80
75
  # 上传文件
81
- with yaspin(Spinners.dots, text="正在上传文件...") as spinner:
82
- try:
83
- with spinner.hidden():
84
- upload_result = platform.upload_files(valid_files)
85
- if not upload_result:
86
- spinner.text = "文件上传失败"
87
- spinner.fail("❌")
88
- return {
89
- "success": False,
90
- "stdout": "",
91
- "stderr": "文件上传失败",
92
- }
93
- spinner.text = "文件上传成功"
94
- spinner.ok("✅")
95
- except Exception as e:
96
- spinner.text = "文件上传失败"
97
- spinner.fail("❌")
76
+ print(f"📤 正在上传文件...")
77
+ try:
78
+ upload_result = platform.upload_files(valid_files)
79
+ if not upload_result:
80
+ print(f"❌ 文件上传失败")
98
81
  return {
99
82
  "success": False,
100
83
  "stdout": "",
101
- "stderr": f"文件上传失败: {str(e)}",
84
+ "stderr": "文件上传失败",
102
85
  }
86
+ print(f"✅ 文件上传成功")
87
+ except Exception as e:
88
+ print(f"❌ 文件上传失败: {str(e)}")
89
+ return {
90
+ "success": False,
91
+ "stdout": "",
92
+ "stderr": f"文件上传失败: {str(e)}",
93
+ }
103
94
 
104
95
  platform.set_suppress_output(False)
105
96
 
@@ -111,11 +102,9 @@ class FileAnalyzerTool:
111
102
  请提供详细的分析结果和理由。"""
112
103
 
113
104
  # 发送请求并获取分析结果
114
- with yaspin(Spinners.dots, text="正在分析文件...") as spinner:
115
- with spinner.hidden():
116
- analysis_result = platform.chat_until_success(analysis_request)
117
- spinner.text = "分析完成"
118
- spinner.ok("✅")
105
+ print(f"🔍 正在分析文件...")
106
+ analysis_result = platform.chat_until_success(analysis_request)
107
+ print(f"✅ 分析完成")
119
108
 
120
109
  # 清理会话
121
110
  platform.delete_chat()
@@ -2,8 +2,6 @@
2
2
  import os
3
3
  from typing import Any, Dict
4
4
 
5
- from yaspin import yaspin
6
-
7
5
  from jarvis.jarvis_utils.output import OutputType, PrettyOutput
8
6
 
9
7
 
@@ -46,89 +44,90 @@ class ReadCodeTool:
46
44
  """
47
45
  try:
48
46
  abs_path = os.path.abspath(filepath)
49
- with yaspin(text=f"正在读取文件: {abs_path}...", color="cyan") as spinner:
50
- # 文件存在性检查
51
- if not os.path.exists(abs_path):
52
- return {
53
- "success": False,
54
- "stdout": "",
55
- "stderr": f"文件不存在: {abs_path}",
56
- }
57
-
58
- # 文件大小限制检查(10MB)
59
- if os.path.getsize(abs_path) > 10 * 1024 * 1024:
60
- return {
61
- "success": False,
62
- "stdout": "",
63
- "stderr": "文件过大 (>10MB)",
64
- }
65
-
66
- # 读取文件内容
67
- with open(abs_path, "r", encoding="utf-8", errors="ignore") as f:
68
- lines = f.readlines()
69
-
70
- total_lines = len(lines)
71
-
72
- # 处理空文件情况
73
- if total_lines == 0:
74
- spinner.ok("✅")
75
- return {
76
- "success": True,
77
- "stdout": f"\n🔍 文件: {abs_path}\n📄 文件为空 (0行)\n",
78
- "stderr": "",
79
- }
80
-
81
- # 处理特殊值-1表示文件末尾
82
- if end_line == -1:
83
- end_line = total_lines
84
- else:
85
- end_line = (
86
- max(1, min(end_line, total_lines))
87
- if end_line >= 0
88
- else total_lines + end_line + 1
89
- )
90
-
91
- start_line = (
92
- max(1, min(start_line, total_lines))
93
- if start_line >= 0
94
- else total_lines + start_line + 1
95
- )
47
+ print(f"📖 正在读取文件: {abs_path}...")
48
+ # 文件存在性检查
49
+ if not os.path.exists(abs_path):
50
+ return {
51
+ "success": False,
52
+ "stdout": "",
53
+ "stderr": f"文件不存在: {abs_path}",
54
+ }
55
+
56
+ # 文件大小限制检查(10MB)
57
+ if os.path.getsize(abs_path) > 10 * 1024 * 1024:
58
+ return {
59
+ "success": False,
60
+ "stdout": "",
61
+ "stderr": "文件过大 (>10MB)",
62
+ }
63
+
64
+ # 读取文件内容
65
+ with open(abs_path, "r", encoding="utf-8", errors="ignore") as f:
66
+ lines = f.readlines()
96
67
 
97
- if start_line > end_line:
98
- spinner.fail("❌")
99
- return {
100
- "success": False,
101
- "stdout": "",
102
- "stderr": f"无效的行范围 [{start_line}-{end_line}] (总行数: {total_lines})",
103
- }
104
-
105
- # 添加行号并构建输出内容
106
- selected_lines = lines[start_line - 1 : end_line]
107
- numbered_content = "".join(
108
- [
109
- f"{i:4d}:{line}"
110
- for i, line in enumerate(selected_lines, start=start_line)
111
- ]
68
+ total_lines = len(lines)
69
+
70
+ # 处理空文件情况
71
+ if total_lines == 0:
72
+ print(f" 文件读取完成: {abs_path}")
73
+ return {
74
+ "success": True,
75
+ "stdout": f"\n🔍 文件: {abs_path}\n📄 文件为空 (0行)\n",
76
+ "stderr": "",
77
+ }
78
+
79
+ # 处理特殊值-1表示文件末尾
80
+ if end_line == -1:
81
+ end_line = total_lines
82
+ else:
83
+ end_line = (
84
+ max(1, min(end_line, total_lines))
85
+ if end_line >= 0
86
+ else total_lines + end_line + 1
112
87
  )
113
88
 
114
- # 构建输出格式
115
- output = (
116
- f"\n🔍 文件: {abs_path}\n"
117
- f"📄 原始行号: {start_line}-{end_line} (共{total_lines}行) \n\n"
118
- f"{numbered_content}\n\n"
89
+ start_line = (
90
+ max(1, min(start_line, total_lines))
91
+ if start_line >= 0
92
+ else total_lines + start_line + 1
93
+ )
94
+
95
+ if start_line > end_line:
96
+ print(
97
+ f"❌ 无效的行范围 [{start_line}-{end_line}] (总行数: {total_lines})"
119
98
  )
120
- spinner.text = f"文件读取完成: {abs_path}"
121
- spinner.ok("")
122
-
123
- if agent:
124
- files = agent.get_user_data("files")
125
- if files:
126
- files.append(abs_path)
127
- else:
128
- files = [abs_path]
129
- agent.set_user_data("files", files)
130
-
131
- return {"success": True, "stdout": output, "stderr": ""}
99
+ return {
100
+ "success": False,
101
+ "stdout": "",
102
+ "stderr": f"无效的行范围 [{start_line}-{end_line}] (总行数: {total_lines})",
103
+ }
104
+
105
+ # 添加行号并构建输出内容
106
+ selected_lines = lines[start_line - 1 : end_line]
107
+ numbered_content = "".join(
108
+ [
109
+ f"{i:4d}:{line}"
110
+ for i, line in enumerate(selected_lines, start=start_line)
111
+ ]
112
+ )
113
+
114
+ # 构建输出格式
115
+ output = (
116
+ f"\n🔍 文件: {abs_path}\n"
117
+ f"📄 原始行号: {start_line}-{end_line} (共{total_lines}行) \n\n"
118
+ f"{numbered_content}\n\n"
119
+ )
120
+ print(f"✅ 文件读取完成: {abs_path}")
121
+
122
+ if agent:
123
+ files = agent.get_user_data("files")
124
+ if files:
125
+ files.append(abs_path)
126
+ else:
127
+ files = [abs_path]
128
+ agent.set_user_data("files", files)
129
+
130
+ return {"success": True, "stdout": output, "stderr": ""}
132
131
 
133
132
  except Exception as e:
134
133
  PrettyOutput.print(str(e), OutputType.ERROR)
@@ -29,42 +29,7 @@ arguments:
29
29
  """,
30
30
  "description": "网页搜索",
31
31
  },
32
- "Plan": {
33
- "append": True,
34
- "template": f"""
35
- 请使用code_plan工具生成代码修改计划,必须严格遵守以下工具调用格式:
36
-
37
- {ot("TOOL_CALL")}
38
- want: 想要从执行结果中获取到的信息
39
- name: code_plan
40
- arguments:
41
- requirement: "需要实现用户登录功能,包括用户名密码验证和JWT生成"
42
- {ct("TOOL_CALL")}
43
-
44
- 请提供详细的需求描述和完整上下文信息:
45
-
46
- 必须包含的上下文信息:
47
- 1. 当前会话状态 - 当前正在处理的任务和进度
48
- 2. 用户历史请求 - 与本任务相关的历史请求
49
- 3. 系统状态 - 相关的系统配置和环境状态
50
- 4. 已确定的相关文件 - 已经分析出的需要修改的文件列表
51
-
52
- 示例:
53
- 1. 需要实现用户登录功能,包括用户名密码验证和JWT生成
54
- 2. 重构订单处理模块以提高性能
55
- 3. 优化数据库查询性能,减少响应时间
56
- 4. 添加支付网关集成功能
57
- 5. 修改用户权限管理系统
58
-
59
- code_plan工具将:
60
- 1. 分析项目结构确定相关文件
61
- 2. 理解需求后制定详细修改步骤
62
- 3. 按功能模块分组修改内容
63
- 4. 评估修改影响范围
64
- 5. 生成可执行的开发计划
65
- """,
66
- "description": "生成代码修改计划",
67
- },
32
+
68
33
  "FindRelatedFiles": {
69
34
  "append": False,
70
35
  "template": f"""
@@ -3,7 +3,7 @@ import os
3
3
  from functools import lru_cache
4
4
  from typing import Any, Dict, List
5
5
 
6
- import yaml
6
+ import yaml # type: ignore
7
7
 
8
8
  from jarvis.jarvis_utils.builtin_replace_map import BUILTIN_REPLACE_MAP
9
9
 
@@ -96,25 +96,21 @@ def get_max_input_token_count() -> int:
96
96
  return int(GLOBAL_CONFIG_DATA.get("JARVIS_MAX_INPUT_TOKEN_COUNT", "32000"))
97
97
 
98
98
 
99
- def is_auto_complete() -> bool:
100
- """
101
- 检查是否启用了自动补全功能。
102
-
103
- 返回:
104
- bool: 如果启用了自动补全则返回True,默认为False
105
- """
106
- return GLOBAL_CONFIG_DATA.get("JARVIS_AUTO_COMPLETE", False) == True
107
-
108
-
109
99
  def get_shell_name() -> str:
110
100
  """
111
101
  获取系统shell名称。
112
102
 
113
103
  返回:
114
- str: Shell名称(例如bash, zsh),默认为bash
104
+ str: Shell名称(例如bash, zsh, fish),默认为bash
105
+
106
+ 获取顺序:
107
+ 1. 先从GLOBAL_CONFIG_DATA中获取JARVIS_SHELL配置
108
+ 2. 再从GLOBAL_CONFIG_DATA中获取SHELL配置
109
+ 3. 最后从环境变量SHELL获取
110
+ 4. 如果都未配置,则默认返回bash
115
111
  """
116
- shell_path = GLOBAL_CONFIG_DATA.get("SHELL", "/bin/bash")
117
- return os.path.basename(shell_path)
112
+ shell_path = GLOBAL_CONFIG_DATA.get("SHELL", os.getenv("SHELL", "/bin/bash"))
113
+ return os.path.basename(shell_path).lower()
118
114
 
119
115
 
120
116
  def get_normal_platform_name() -> str:
@@ -181,16 +177,6 @@ def is_confirm_before_apply_patch() -> bool:
181
177
  return GLOBAL_CONFIG_DATA.get("JARVIS_CONFIRM_BEFORE_APPLY_PATCH", False) == True
182
178
 
183
179
 
184
- def get_max_tool_call_count() -> int:
185
- """
186
- 获取最大工具调用次数。
187
-
188
- 返回:
189
- int: 最大连续工具调用次数,默认为20
190
- """
191
- return int(GLOBAL_CONFIG_DATA.get("JARVIS_MAX_TOOL_CALL_COUNT", "20"))
192
-
193
-
194
180
  def get_data_dir() -> str:
195
181
  """
196
182
  获取Jarvis数据存储目录路径。
@@ -199,20 +185,9 @@ def get_data_dir() -> str:
199
185
  str: 数据目录路径,优先从JARVIS_DATA_PATH环境变量获取,
200
186
  如果未设置或为空,则使用~/.jarvis作为默认值
201
187
  """
202
- data_path = GLOBAL_CONFIG_DATA.get("JARVIS_DATA_PATH", "").strip()
203
- if not data_path:
204
- return os.path.expanduser("~/.jarvis")
205
- return data_path
206
-
207
-
208
- def get_auto_update() -> bool:
209
- """
210
- 获取是否自动更新git仓库。
211
-
212
- 返回:
213
- bool: 如果需要自动更新则返回True,默认为True
214
- """
215
- return GLOBAL_CONFIG_DATA.get("JARVIS_AUTO_UPDATE", True) == True
188
+ return os.path.expanduser(
189
+ GLOBAL_CONFIG_DATA.get("JARVIS_DATA_PATH", "~/.jarvis").strip()
190
+ )
216
191
 
217
192
 
218
193
  def get_max_big_content_size() -> int:
@@ -265,16 +240,6 @@ def is_print_prompt() -> bool:
265
240
  return GLOBAL_CONFIG_DATA.get("JARVIS_PRINT_PROMPT", False) == True
266
241
 
267
242
 
268
- def get_history_count() -> int:
269
- """
270
- 获取是否启用历史记录功能。
271
-
272
- 返回:
273
- bool: 如果启用历史记录则返回True,默认为False
274
- """
275
- return GLOBAL_CONFIG_DATA.get("JARVIS_USE_HISTORY_COUNT", 0)
276
-
277
-
278
243
  def get_mcp_config() -> List[Dict[str, Any]]:
279
244
  """
280
245
  获取MCP配置列表。
@@ -1,17 +1,11 @@
1
1
  # -*- coding: utf-8 -*-
2
- import functools
3
- import os
4
- from typing import Any, List
2
+ from typing import List
5
3
 
6
- from jarvis.jarvis_utils.config import get_data_dir
7
4
  from jarvis.jarvis_utils.output import OutputType, PrettyOutput
8
5
 
9
- # 全局缓存,避免重复加载模型
10
- _global_tokenizers = {}
11
-
12
6
 
13
7
  def get_context_token_count(text: str) -> int:
14
- """使用分词器获取文本的token数量。
8
+ """使用tiktoken获取文本的token数量。
15
9
 
16
10
  参数:
17
11
  text: 要计算token的输入文本
@@ -20,16 +14,10 @@ def get_context_token_count(text: str) -> int:
20
14
  int: 文本中的token数量
21
15
  """
22
16
  try:
23
- from transformers import AutoTokenizer # type: ignore
24
- tokenizer : AutoTokenizer = load_tokenizer()
25
- # 分批处理长文本,确保不超过模型最大长度
26
- total_tokens = 0
27
- chunk_size = 100 # 每次处理100个字符,避免超过模型最大长度(考虑到中文字符可能被编码成多个token)
28
- for i in range(0, len(text), chunk_size):
29
- chunk = text[i : i + chunk_size]
30
- tokens = tokenizer.encode(chunk) # type: ignore
31
- total_tokens += len(tokens)
32
- return total_tokens
17
+ import tiktoken
18
+
19
+ encoding = tiktoken.get_encoding("cl100k_base")
20
+ return len(encoding.encode(text))
33
21
  except Exception as e:
34
22
  PrettyOutput.print(f"计算token失败: {str(e)}", OutputType.WARNING)
35
23
  return len(text) // 4 # 每个token大约4个字符的粗略估计
@@ -84,36 +72,3 @@ def split_text_into_chunks(
84
72
  PrettyOutput.print(f"文本分割失败: {str(e)}", OutputType.WARNING)
85
73
  # 发生错误时回退到简单的字符分割
86
74
  return [text[i : i + max_length] for i in range(0, len(text), max_length)]
87
-
88
-
89
- @functools.lru_cache(maxsize=1)
90
- def load_tokenizer() -> Any:
91
- """
92
- 加载用于文本处理的分词器,使用缓存避免重复加载。
93
-
94
- 返回:
95
- AutoTokenizer: 加载的分词器
96
- """
97
-
98
- from transformers import AutoTokenizer # type: ignore
99
-
100
- model_name = "gpt2"
101
- cache_dir = os.path.join(get_data_dir(), "huggingface", "hub")
102
-
103
- # 检查全局缓存
104
- if model_name in _global_tokenizers:
105
- return _global_tokenizers[model_name]
106
-
107
- try:
108
- tokenizer = AutoTokenizer.from_pretrained(
109
- model_name, cache_dir=cache_dir, local_files_only=True
110
- )
111
- except Exception:
112
- tokenizer = AutoTokenizer.from_pretrained(
113
- model_name, cache_dir=cache_dir, local_files_only=False
114
- )
115
-
116
- # 保存到全局缓存
117
- _global_tokenizers[model_name] = tokenizer
118
-
119
- return tokenizer # type: ignore