pro-craft 0.1.55__tar.gz → 0.2.57__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pro-craft might be problematic. Click here for more details.
- {pro_craft-0.1.55 → pro_craft-0.2.57}/PKG-INFO +1 -1
- {pro_craft-0.1.55 → pro_craft-0.2.57}/pyproject.toml +1 -1
- pro_craft-0.2.57/src/pro_craft/__init__.py +10 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/database.py +30 -89
- pro_craft-0.2.57/src/pro_craft/log.py +111 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/prompt_craft/async_.py +193 -31
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/server/router/prompt.py +19 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft.egg-info/PKG-INFO +1 -1
- pro_craft-0.1.55/src/pro_craft/__init__.py +0 -18
- pro_craft-0.1.55/src/pro_craft/log.py +0 -82
- {pro_craft-0.1.55 → pro_craft-0.2.57}/README.md +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/setup.cfg +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/code_helper/coder.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/code_helper/designer.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/file_manager.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/prompt_craft/__init__.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/prompt_craft/new.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/prompt_craft/sync.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/server/mcp/__init__.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/server/mcp/models.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/server/mcp/prompt.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/server/router/__init__.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/server/router/models.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft/utils.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft.egg-info/SOURCES.txt +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft.egg-info/dependency_links.txt +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft.egg-info/requires.txt +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/src/pro_craft.egg-info/top_level.txt +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/tests/test22.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/tests/test_11.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/tests/test_coder.py +0 -0
- {pro_craft-0.1.55 → pro_craft-0.2.57}/tests/test_designer.py +0 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from dotenv import load_dotenv, find_dotenv
|
|
2
|
+
dotenv_path = find_dotenv()
|
|
3
|
+
load_dotenv(".env", override=True)
|
|
4
|
+
|
|
5
|
+
from .log import Log
|
|
6
|
+
Log_ = Log(console_level = 30, # 显示控制台的等级 WARNING
|
|
7
|
+
log_file_name="app.log")
|
|
8
|
+
logger = Log_.logger
|
|
9
|
+
|
|
10
|
+
from .prompt_craft import AsyncIntel, Intel, IntelNew
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
|
|
2
|
-
from sqlalchemy import Column, Integer, String, Text, DateTime, text, UniqueConstraint, Boolean, func, Float
|
|
2
|
+
from sqlalchemy import Column, Integer, String, Text, DateTime, text, UniqueConstraint, Boolean, func, Float, Double
|
|
3
3
|
from sqlalchemy.orm import declarative_base
|
|
4
4
|
|
|
5
5
|
from datetime import datetime, timedelta
|
|
@@ -179,13 +179,10 @@ class Prompt(PromptBase):
|
|
|
179
179
|
class UseCase(PromptBase):
|
|
180
180
|
__tablename__ = 'ai_usecase' # 数据库中的表名,你可以改成你希望的名字
|
|
181
181
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
# id (int, primary_key=True, autoincrement=True)
|
|
188
|
-
# 你的属性表中 id 为 int, true (not null), true (primary key), 0 (length), ASC (key order), true (auto increment)
|
|
182
|
+
__table_args__ = (
|
|
183
|
+
UniqueConstraint('time',name='time_double_uc'),
|
|
184
|
+
# 'name' 参数是可选的,用于给数据库中的约束指定一个名称,方便管理和调试
|
|
185
|
+
)
|
|
189
186
|
id = Column(
|
|
190
187
|
Integer,
|
|
191
188
|
primary_key=True,
|
|
@@ -196,19 +193,16 @@ class UseCase(PromptBase):
|
|
|
196
193
|
|
|
197
194
|
# prompt_id (varchar 255, not null, unique)
|
|
198
195
|
# 你的属性表中 prompt_id 为 varchar, 255 (length), true (not null)
|
|
199
|
-
|
|
196
|
+
|
|
197
|
+
level = Column(
|
|
200
198
|
String(255), # VARCHAR 类型,长度 255
|
|
201
199
|
nullable=False, # 不能为空 # 必须是唯一的,这会创建唯一索引
|
|
202
|
-
comment="
|
|
200
|
+
comment="level"
|
|
203
201
|
)
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
use_case = Column(
|
|
209
|
-
Text, # TEXT 类型,适用于长文本
|
|
210
|
-
nullable=False, # 不能为空
|
|
211
|
-
comment="用例"
|
|
202
|
+
time = Column(
|
|
203
|
+
Double,
|
|
204
|
+
nullable=False, # 不能为空
|
|
205
|
+
comment="时间戳"
|
|
212
206
|
)
|
|
213
207
|
|
|
214
208
|
timestamp = Column(
|
|
@@ -219,101 +213,48 @@ class UseCase(PromptBase):
|
|
|
219
213
|
comment="时间戳"
|
|
220
214
|
)
|
|
221
215
|
|
|
222
|
-
|
|
223
|
-
|
|
216
|
+
filepath = Column(
|
|
217
|
+
String(255), # TEXT 类型,适用于长文本
|
|
224
218
|
nullable=False, # 不能为空
|
|
225
|
-
comment="
|
|
219
|
+
comment="文件路径"
|
|
226
220
|
)
|
|
227
221
|
|
|
228
|
-
# use_case (text, nullable)
|
|
229
|
-
# 你的属性表中 use_case 为 text, false (not null, 即 nullable=True), NULL (default value), '用例' (comment)
|
|
230
|
-
solution = Column(
|
|
231
|
-
Text,
|
|
232
|
-
nullable=True, # 可以为空 (因为你的表格中 Not Null 为 false)
|
|
233
|
-
comment="理想结果" # 列注释
|
|
234
|
-
)
|
|
235
222
|
|
|
236
|
-
|
|
237
|
-
|
|
223
|
+
function = Column(
|
|
224
|
+
String(255), # TEXT 类型,适用于长文本
|
|
238
225
|
nullable=False, # 不能为空
|
|
239
|
-
comment="
|
|
226
|
+
comment="函数"
|
|
240
227
|
)
|
|
241
228
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
# 定义 __repr__ 方法以便打印对象时有清晰的表示
|
|
245
|
-
def __repr__(self):
|
|
246
|
-
return (f"<Prompt(id={self.id}, prompt_id='{self.prompt_id}', "
|
|
247
|
-
f"use_case='{self.use_case[:50]}...', solution='{self.solution}')>"
|
|
248
|
-
f"is_deleted='{self.is_deleted}...'>")
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
class EvalsInfo(PromptBase):
|
|
252
|
-
__tablename__ = 'ai_evals' # 数据库中的表名,你可以改成你希望的名字
|
|
253
|
-
|
|
254
|
-
# __table_args__ = (
|
|
255
|
-
# UniqueConstraint('prompt_id', 'use_case', name='_prompt_id_version_uc'),
|
|
256
|
-
# # 'name' 参数是可选的,用于给数据库中的约束指定一个名称,方便管理和调试
|
|
257
|
-
# )
|
|
258
|
-
|
|
259
|
-
# id (int, primary_key=True, autoincrement=True)
|
|
260
|
-
# 你的属性表中 id 为 int, true (not null), true (primary key), 0 (length), ASC (key order), true (auto increment)
|
|
261
|
-
id = Column(
|
|
262
|
-
Integer,
|
|
263
|
-
primary_key=True,
|
|
264
|
-
autoincrement=True, # 自动递增
|
|
229
|
+
lines = Column(
|
|
230
|
+
String(255), # TEXT 类型,适用于长文本
|
|
265
231
|
nullable=False, # 不能为空
|
|
266
|
-
comment="
|
|
267
|
-
)
|
|
268
|
-
|
|
269
|
-
# prompt_id (varchar 255, not null, unique)
|
|
270
|
-
# 你的属性表中 prompt_id 为 varchar, 255 (length), true (not null)
|
|
271
|
-
prompt_id = Column(
|
|
272
|
-
String(255), # VARCHAR 类型,长度 255
|
|
273
|
-
nullable=False, # 不能为空 # 必须是唯一的,这会创建唯一索引
|
|
274
|
-
comment="Unique identifier for the prompt"
|
|
275
|
-
)
|
|
276
|
-
status = Column(
|
|
277
|
-
String(255), # VARCHAR 类型,长度 255
|
|
278
|
-
nullable=False, # 不能为空 # 必须是唯一的,这会创建唯一索引
|
|
279
|
-
comment="通过/未通过"
|
|
232
|
+
comment="函数"
|
|
280
233
|
)
|
|
281
234
|
|
|
282
|
-
|
|
283
|
-
|
|
235
|
+
type_ = Column(
|
|
236
|
+
String(255), # TEXT 类型,适用于长文本
|
|
284
237
|
nullable=False, # 不能为空
|
|
285
|
-
comment="
|
|
238
|
+
comment="函数"
|
|
286
239
|
)
|
|
287
240
|
|
|
288
|
-
|
|
289
|
-
|
|
241
|
+
target = Column(
|
|
242
|
+
Text, # TEXT 类型,适用于长文本
|
|
290
243
|
nullable=False, # 不能为空
|
|
291
|
-
comment="
|
|
244
|
+
comment="标签"
|
|
292
245
|
)
|
|
293
246
|
|
|
294
|
-
|
|
295
|
-
# 你的属性表中 prompt 为 text, true (not null)
|
|
296
|
-
bad_case = Column(
|
|
247
|
+
content = Column(
|
|
297
248
|
Text, # TEXT 类型,适用于长文本
|
|
298
249
|
nullable=False, # 不能为空
|
|
299
|
-
comment="
|
|
300
|
-
)
|
|
301
|
-
|
|
302
|
-
timestamp = Column(
|
|
303
|
-
DateTime,
|
|
304
|
-
nullable=False, # 不能为空
|
|
305
|
-
server_default=text('CURRENT_TIMESTAMP'),
|
|
306
|
-
onupdate=text('CURRENT_TIMESTAMP'),
|
|
307
|
-
comment="时间戳"
|
|
250
|
+
comment="The actual prompt text content"
|
|
308
251
|
)
|
|
309
252
|
|
|
310
|
-
|
|
311
253
|
is_deleted = Column(Boolean, default=False, server_default=text('0'))
|
|
312
254
|
|
|
313
255
|
# 定义 __repr__ 方法以便打印对象时有清晰的表示
|
|
314
256
|
def __repr__(self):
|
|
315
257
|
return (f"<Prompt(id={self.id}, prompt_id='{self.prompt_id}', "
|
|
316
|
-
f"
|
|
317
|
-
f"total='{self.total}...', score='{self.score}')>"
|
|
258
|
+
f"use_case='{self.use_case[:50]}...', solution='{self.solution}')>"
|
|
318
259
|
f"is_deleted='{self.is_deleted}...'>")
|
|
319
260
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
|
|
4
|
+
import inspect
|
|
5
|
+
class Log:
|
|
6
|
+
_instance = None
|
|
7
|
+
def __new__(cls, *args, **kwargs):
|
|
8
|
+
if cls._instance is None:
|
|
9
|
+
cls._instance = super().__new__(cls)
|
|
10
|
+
return cls._instance
|
|
11
|
+
|
|
12
|
+
def __init__(self, console_level = logging.INFO, log_file_name="app.log"):
|
|
13
|
+
self.Console_LOG_LEVEL = console_level
|
|
14
|
+
self.log_file_name = log_file_name
|
|
15
|
+
self.LOG_FILE_PATH = os.path.join("logs", log_file_name)
|
|
16
|
+
self.logger = self.get_logger()
|
|
17
|
+
|
|
18
|
+
def get_logger(self):
|
|
19
|
+
logger = logging.getLogger()
|
|
20
|
+
# logger.setLevel(self.LOG_LEVEL)
|
|
21
|
+
if not logger.handlers:
|
|
22
|
+
# --- 4. 配置 Formatter (格式化器) ---
|
|
23
|
+
# 以后有一个标准化的日志要使用logger 而非标的则使用super-log
|
|
24
|
+
formatter = logging.Formatter(
|
|
25
|
+
"%(asctime)s $ %(levelname)s $ %(created)f $ %(pathname)s $ %(funcName)s$ :%(lineno)d $ %(message)s||"
|
|
26
|
+
)
|
|
27
|
+
# --- 5. 配置 Handler (处理器) ---
|
|
28
|
+
|
|
29
|
+
# 5.1 控制台处理器 (StreamHandler)
|
|
30
|
+
console_handler = logging.StreamHandler()
|
|
31
|
+
console_handler.setLevel(self.Console_LOG_LEVEL) # 控制台只显示 INFO 及以上级别的日志
|
|
32
|
+
console_handler.setFormatter(formatter)
|
|
33
|
+
logger.addHandler(console_handler)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# 文件系统
|
|
37
|
+
## 主日志本
|
|
38
|
+
file_handler = RotatingFileHandler( # RotatingFileHandler: 按文件大小轮转
|
|
39
|
+
self.LOG_FILE_PATH,
|
|
40
|
+
maxBytes=10 * 1024 * 1024, # 10 MB # maxBytes: 单个日志文件的最大字节数 (例如 10MB)
|
|
41
|
+
backupCount=10, # backupCount: 保留的旧日志文件数量
|
|
42
|
+
encoding="utf-8",
|
|
43
|
+
)
|
|
44
|
+
file_handler.setLevel(logging.DEBUG) # 文件中显示所有指定级别的日志
|
|
45
|
+
file_handler.setFormatter(formatter)
|
|
46
|
+
logger.addHandler(file_handler)
|
|
47
|
+
|
|
48
|
+
## 运行日志本
|
|
49
|
+
file_handler_info = RotatingFileHandler(
|
|
50
|
+
self.LOG_FILE_PATH.replace('.log','_info.log'),
|
|
51
|
+
maxBytes=10 * 1024 * 1024, # 10 MB
|
|
52
|
+
backupCount=5,
|
|
53
|
+
encoding="utf-8",
|
|
54
|
+
)
|
|
55
|
+
file_handler_info.setLevel(logging.INFO) # 文件中显示所有指定级别的日志
|
|
56
|
+
file_handler_info.setFormatter(formatter)
|
|
57
|
+
logger.addHandler(file_handler_info)
|
|
58
|
+
|
|
59
|
+
## 错误日志本
|
|
60
|
+
file_handler_warning = RotatingFileHandler(
|
|
61
|
+
self.LOG_FILE_PATH.replace('.log','_err.log'),
|
|
62
|
+
maxBytes=10 * 1024 * 1024, # 10 MB
|
|
63
|
+
backupCount=5,
|
|
64
|
+
encoding="utf-8",
|
|
65
|
+
)
|
|
66
|
+
file_handler_warning.setLevel(logging.WARNING) # 文件中显示所有指定级别的日志
|
|
67
|
+
file_handler_warning.setFormatter(formatter)
|
|
68
|
+
logger.addHandler(file_handler_warning)
|
|
69
|
+
|
|
70
|
+
## 指定日志本
|
|
71
|
+
file_handler_super = RotatingFileHandler(
|
|
72
|
+
self.LOG_FILE_PATH.replace('.log','_s.log'),
|
|
73
|
+
maxBytes=5 * 1024 * 1024, # 10 MB
|
|
74
|
+
backupCount=5,
|
|
75
|
+
encoding="utf-8",
|
|
76
|
+
)
|
|
77
|
+
file_handler_super.setLevel(logging.CRITICAL) # 文件中显示所有指定级别的日志
|
|
78
|
+
file_handler_super.setFormatter(formatter)
|
|
79
|
+
logger.addHandler(file_handler_super)
|
|
80
|
+
|
|
81
|
+
# class ConfigFilter(logging.Filter):
|
|
82
|
+
# # 初始化的方案
|
|
83
|
+
# def __init__(self, config_value, name=''):
|
|
84
|
+
# super().__init__(name)
|
|
85
|
+
# self.config_value = config_value
|
|
86
|
+
|
|
87
|
+
# def filter(self, record):
|
|
88
|
+
# record.config_info = self.config_value # 添加到LogRecord
|
|
89
|
+
# return True
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# class CustomFilter(logging.Filter):
|
|
93
|
+
# def filter(self, record):
|
|
94
|
+
# # 添加自定义的用户ID
|
|
95
|
+
# print(record,'recordrecordrecordrecordrecord')
|
|
96
|
+
# record.user_id = os.getenv("CURRENT_USER_ID", "anonymous")
|
|
97
|
+
# # 添加会话ID
|
|
98
|
+
# # record.session_id = "SESSION_XYZ123" # 实际应用中可能从request或全局变量获取
|
|
99
|
+
# frame = inspect.currentframe()
|
|
100
|
+
# info = inspect.getframeinfo(frame)
|
|
101
|
+
# # logger.error(f"Function name: {info.function} : {e}")
|
|
102
|
+
# record.session_id = f"Function name: {info.function}"
|
|
103
|
+
|
|
104
|
+
# return True # 必须返回True才能继续处理该日志记录
|
|
105
|
+
|
|
106
|
+
# # 将自定义过滤器添加到处理器或记录器
|
|
107
|
+
# logger.addFilter(CustomFilter())
|
|
108
|
+
# handler.addFilter(CustomFilter()) # 也可以加到handler上
|
|
109
|
+
|
|
110
|
+
logger.info("这是一个包含自定义信息的日志")
|
|
111
|
+
return logger
|
|
@@ -26,14 +26,12 @@ from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sess
|
|
|
26
26
|
from sqlalchemy import select, and_ # 引入 select 和 and_
|
|
27
27
|
from sqlalchemy.orm import class_mapper # 用于检查对象是否是持久化的
|
|
28
28
|
from datetime import datetime, timedelta
|
|
29
|
-
|
|
29
|
+
import datetime
|
|
30
30
|
from tqdm.asyncio import tqdm
|
|
31
31
|
import pandas as pd
|
|
32
32
|
import plotly.graph_objects as go
|
|
33
|
-
from pro_craft import super_log
|
|
34
33
|
|
|
35
|
-
BATCH_SIZE = int(os.getenv("DATABASE_SYNC_BATCH_SIZE",
|
|
36
|
-
print(12)
|
|
34
|
+
BATCH_SIZE = int(os.getenv("DATABASE_SYNC_BATCH_SIZE",1000))
|
|
37
35
|
|
|
38
36
|
def fix_broken_json_string(broken_json_str):
|
|
39
37
|
# 移除 BOM
|
|
@@ -119,7 +117,6 @@ class AsyncIntel():
|
|
|
119
117
|
assert database_url
|
|
120
118
|
assert 'aio' in database_url
|
|
121
119
|
except AssertionError as e:
|
|
122
|
-
super_log(database_url,'database_url',logger=self.logger.warning)
|
|
123
120
|
raise IntellectRemoveFormatError(f"异步服务url必须提供, 且必须是aiomysql配置") from e
|
|
124
121
|
|
|
125
122
|
self.engine = create_async_engine(database_url, echo=False,
|
|
@@ -287,7 +284,6 @@ class AsyncIntel():
|
|
|
287
284
|
if prompt_obj:
|
|
288
285
|
return prompt_obj
|
|
289
286
|
|
|
290
|
-
print("warnning 未找到制定版本, 默认使用最新版本")
|
|
291
287
|
prompt_obj = await self.get_prompt(prompt_id=prompt_id,version=None,session=session)
|
|
292
288
|
return prompt_obj
|
|
293
289
|
|
|
@@ -354,6 +350,170 @@ class AsyncIntel():
|
|
|
354
350
|
use_case = result.scalars().all()
|
|
355
351
|
return use_case
|
|
356
352
|
|
|
353
|
+
async def save_use_case2(self,session = None):
|
|
354
|
+
with open("/Users/zhaoxuefeng/GitHub/digital_life/logs/app.log",'r') as f:
|
|
355
|
+
x = f.read()
|
|
356
|
+
|
|
357
|
+
def work(resu):
|
|
358
|
+
if len(resu) == 9:
|
|
359
|
+
unix_timestamp_str = resu[2]
|
|
360
|
+
dt_object = datetime.datetime.fromtimestamp(float(unix_timestamp_str.strip()))
|
|
361
|
+
use_case = UseCase(
|
|
362
|
+
time = resu[2],
|
|
363
|
+
level = resu[1],
|
|
364
|
+
timestamp =dt_object.strftime('%Y-%m-%d %H:%M:%S.%f'),
|
|
365
|
+
filepath=resu[3],
|
|
366
|
+
function=resu[4],
|
|
367
|
+
lines=resu[5],
|
|
368
|
+
type_=resu[6],
|
|
369
|
+
target=resu[7],
|
|
370
|
+
content=resu[8],
|
|
371
|
+
)
|
|
372
|
+
session.add(use_case)
|
|
373
|
+
else:
|
|
374
|
+
print(len(resu))
|
|
375
|
+
print(resu,'resu')
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
for res in x.split("||"):
|
|
379
|
+
resu = res.split("$")
|
|
380
|
+
work(resu)
|
|
381
|
+
|
|
382
|
+
await session.commit() # 提交事务,将数据写入数据库
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
async def save_use_case3(self, session):
|
|
386
|
+
log_filepath = "/Users/zhaoxuefeng/GitHub/digital_life/logs/app.log"
|
|
387
|
+
|
|
388
|
+
# 1. 获取数据库中已有的最新时间戳
|
|
389
|
+
# 假设 timestamp 列是 DATETIME 类型,且是用于唯一标识和排序的关键字段
|
|
390
|
+
latest_db_timestamp = None
|
|
391
|
+
try:
|
|
392
|
+
result = await session.execute(
|
|
393
|
+
select(UseCase.timestamp)
|
|
394
|
+
.order_by(UseCase.timestamp.desc())
|
|
395
|
+
.limit(1)
|
|
396
|
+
)
|
|
397
|
+
latest_db_timestamp = result.scalar_one_or_none()
|
|
398
|
+
if latest_db_timestamp:
|
|
399
|
+
print(f"Latest timestamp in DB: {latest_db_timestamp}")
|
|
400
|
+
else:
|
|
401
|
+
print("No records found in DB. Starting fresh.")
|
|
402
|
+
except Exception as e:
|
|
403
|
+
print(f"Error querying latest timestamp: {e}")
|
|
404
|
+
# 如果查询失败,可以选择继续,但可能导致重复导入,或者直接退出
|
|
405
|
+
return
|
|
406
|
+
|
|
407
|
+
added_count = 0
|
|
408
|
+
skipped_count = 0
|
|
409
|
+
error_count = 0
|
|
410
|
+
|
|
411
|
+
# 2. 读取并处理日志文件
|
|
412
|
+
try:
|
|
413
|
+
with open(log_filepath, 'r') as f:
|
|
414
|
+
x = f.read()
|
|
415
|
+
except FileNotFoundError:
|
|
416
|
+
print(f"Error: Log file not found at {log_filepath}")
|
|
417
|
+
return
|
|
418
|
+
|
|
419
|
+
# 日志记录通常是逐行添加的,所以倒序处理可能更高效,但也取决于文件大小和格式
|
|
420
|
+
# 对于你当前的分隔符格式,还是顺序处理比较直接
|
|
421
|
+
for res_str in x.split("||"):
|
|
422
|
+
if not res_str.strip(): # 跳过空字符串
|
|
423
|
+
continue
|
|
424
|
+
|
|
425
|
+
# 使用 try-except 块来处理可能的解析错误
|
|
426
|
+
try:
|
|
427
|
+
resu = res_str.split("$")
|
|
428
|
+
# 检查字段数量是否正确
|
|
429
|
+
# 你的原始代码期望 len(resu) == 9, 但是 SQL 语句有 10 个字段,
|
|
430
|
+
# UseCase 构造函数也有 9 个参数,这需要对应起来
|
|
431
|
+
# level, time, timestamp, filepath, function, lines, type_, target, content, is_deleted
|
|
432
|
+
# 对应 resu 的 index: [1], [2], [2], [3], [4], [5], [6], [7], [8]
|
|
433
|
+
# 看起来 resu[0] 是空的或者不用的,所以 resu 至少需要有 9 个元素(索引0-8)
|
|
434
|
+
# 因此,判断 len(resu) >= 9 即可
|
|
435
|
+
if len(resu) < 9:
|
|
436
|
+
print(f"Skipping malformed log entry (not enough fields): {res_str}")
|
|
437
|
+
error_count += 1
|
|
438
|
+
continue
|
|
439
|
+
|
|
440
|
+
# 提取并清理原始数据
|
|
441
|
+
level_raw = resu[0].strip() # 假设 resu[0] 是 level
|
|
442
|
+
time_raw = resu[1].strip() # 假设 resu[1] 是 time (原始日志时间字符串)
|
|
443
|
+
timestamp_raw = resu[2].strip() # 假设 resu[2] 是 timestamp (unix时间戳字符串)
|
|
444
|
+
filepath_raw = resu[3].strip()
|
|
445
|
+
function_raw = resu[4].strip()
|
|
446
|
+
lines_raw = resu[5].strip()
|
|
447
|
+
type_raw = resu[6].strip()
|
|
448
|
+
target_raw = resu[7].strip()
|
|
449
|
+
content_raw = resu[8].strip()
|
|
450
|
+
|
|
451
|
+
# 处理 time 字段 (原始日志时间字符串)
|
|
452
|
+
# 上次错误是 time 列太短,并且有换行符。在这里清理一下。
|
|
453
|
+
# 假设 time 字段就是你日志中 YYYY-MM-DD HH:MM:SS,ms 这种格式,
|
|
454
|
+
# 但你上次给的示例是 '\n2025-11-01 11:17:52,029 ',所以需要清理
|
|
455
|
+
processed_time_str = time_raw.replace('\n', '').strip()
|
|
456
|
+
# 如果数据库 time 列是 VARCHAR,确保长度够用
|
|
457
|
+
# 如果是 DATETIME,你需要解析它
|
|
458
|
+
# 例如:
|
|
459
|
+
# try:
|
|
460
|
+
# dt_obj_from_time = datetime.datetime.strptime(processed_time_str, '%Y-%m-%d %H:%M:%S,%f')
|
|
461
|
+
# # 再次格式化,确保数据库兼容
|
|
462
|
+
# processed_time_str = dt_obj_from_time.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
|
|
463
|
+
# except ValueError:
|
|
464
|
+
# print(f"Warning: Could not parse 'time' string: {processed_time_str}. Using raw string.")
|
|
465
|
+
# # 或者跳过此条记录,或者设置为 None
|
|
466
|
+
|
|
467
|
+
# 处理 timestamp 字段 (Unix时间戳转换为 datetime 对象)
|
|
468
|
+
try:
|
|
469
|
+
unix_timestamp_float = float(timestamp_raw)
|
|
470
|
+
dt_object = datetime.datetime.fromtimestamp(unix_timestamp_float)
|
|
471
|
+
# 格式化为 MySQL DATETIME(6) 兼容的字符串(包含微秒)
|
|
472
|
+
formatted_timestamp = dt_object.strftime('%Y-%m-%d %H:%M:%S.%f')
|
|
473
|
+
except ValueError:
|
|
474
|
+
print(f"Skipping malformed log entry (invalid timestamp float): {res_str}")
|
|
475
|
+
error_count += 1
|
|
476
|
+
continue
|
|
477
|
+
|
|
478
|
+
# 3. 比较时间戳进行增量检查
|
|
479
|
+
# 将格式化后的时间戳字符串转换为 datetime 对象进行比较
|
|
480
|
+
current_log_timestamp = datetime.datetime.strptime(formatted_timestamp, '%Y-%m-%d %H:%M:%S.%f')
|
|
481
|
+
|
|
482
|
+
if latest_db_timestamp and current_log_timestamp <= latest_db_timestamp:
|
|
483
|
+
# print(f"Skipping existing log entry (timestamp: {current_log_timestamp})")
|
|
484
|
+
skipped_count += 1
|
|
485
|
+
continue # 跳过已存在的或旧的记录
|
|
486
|
+
|
|
487
|
+
# 创建 UseCase 实例
|
|
488
|
+
use_case = UseCase(
|
|
489
|
+
time=processed_time_str, # 使用清理后的原始时间字符串
|
|
490
|
+
level=level_raw,
|
|
491
|
+
timestamp=current_log_timestamp, # 传入 datetime 对象
|
|
492
|
+
filepath=filepath_raw,
|
|
493
|
+
function=function_raw,
|
|
494
|
+
lines=lines_raw,
|
|
495
|
+
type_=type_raw,
|
|
496
|
+
target=target_raw,
|
|
497
|
+
content=content_raw,
|
|
498
|
+
is_deleted=False, # 默认值
|
|
499
|
+
)
|
|
500
|
+
session.add(use_case)
|
|
501
|
+
added_count += 1
|
|
502
|
+
|
|
503
|
+
except Exception as e:
|
|
504
|
+
print(f"Error processing log entry: {res_str}. Error: {e}")
|
|
505
|
+
error_count += 1
|
|
506
|
+
session.rollback() # 如果在添加过程中发生错误,回滚当前批次,避免污染 session
|
|
507
|
+
# 重新开始一个新的事务,或者处理这个错误
|
|
508
|
+
|
|
509
|
+
# 4. 提交事务
|
|
510
|
+
try:
|
|
511
|
+
await session.commit()
|
|
512
|
+
print(f"Log processing complete: Added {added_count} new entries, skipped {skipped_count} existing entries, encountered {error_count} errors.")
|
|
513
|
+
except Exception as e:
|
|
514
|
+
print(f"Error during final commit: {e}")
|
|
515
|
+
await session.rollback()
|
|
516
|
+
|
|
357
517
|
async def save_use_case(self,
|
|
358
518
|
prompt_id: str,
|
|
359
519
|
use_case:str = "",
|
|
@@ -374,14 +534,32 @@ class AsyncIntel():
|
|
|
374
534
|
if use_case == use_case_old.use_case:
|
|
375
535
|
# print("用例已经存在")
|
|
376
536
|
return
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
537
|
+
#time,level, timestamp filepath, function lines, type_, target, content
|
|
538
|
+
with open("/Users/zhaoxuefeng/GitHub/digital_life/logs/app.log",'r') as f:
|
|
539
|
+
x = f.read()
|
|
540
|
+
resu = x.split("||")[14].split("$")
|
|
541
|
+
|
|
542
|
+
use_case = UseCase(
|
|
543
|
+
time = resu[0],
|
|
544
|
+
level = resu[1],
|
|
545
|
+
timestamp =resu[2],
|
|
546
|
+
filepath=resu[3],
|
|
547
|
+
function=resu[4],
|
|
548
|
+
lines=resu[5],
|
|
549
|
+
type_=resu[6],
|
|
550
|
+
target=resu[7],
|
|
551
|
+
content=resu[8],
|
|
552
|
+
)
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
# use_case = UseCase(prompt_id=prompt_id,
|
|
557
|
+
# use_case = use_case,
|
|
558
|
+
# timestamp = timestamp,
|
|
559
|
+
# output = output,
|
|
560
|
+
# solution = solution,
|
|
561
|
+
# faired_time = faired_time,
|
|
562
|
+
# )
|
|
385
563
|
|
|
386
564
|
session.add(use_case)
|
|
387
565
|
await session.commit() # 提交事务,将数据写入数据库
|
|
@@ -426,7 +604,6 @@ class AsyncIntel():
|
|
|
426
604
|
output_format: str,
|
|
427
605
|
prompt_id: str,
|
|
428
606
|
version: str = None,
|
|
429
|
-
inference_save_case = True,
|
|
430
607
|
change_case = False,
|
|
431
608
|
):
|
|
432
609
|
"""
|
|
@@ -448,16 +625,6 @@ class AsyncIntel():
|
|
|
448
625
|
if result_obj.action_type == "inference":
|
|
449
626
|
# 直接推理即可
|
|
450
627
|
ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" + input_)
|
|
451
|
-
if inference_save_case:
|
|
452
|
-
# 设计一个机制, 传输说获取300数据, 那么数据就一直流转获取, 知道300截止
|
|
453
|
-
await self.save_use_case(prompt_id,
|
|
454
|
-
use_case = input_,
|
|
455
|
-
timestamp = datetime.now(),
|
|
456
|
-
output = ai_result,
|
|
457
|
-
solution = output_format,
|
|
458
|
-
faired_time = 0,
|
|
459
|
-
session = session,
|
|
460
|
-
)
|
|
461
628
|
|
|
462
629
|
elif result_obj.action_type == "train":
|
|
463
630
|
assert result_obj.demand # 如果type = train 且 demand 是空 则报错
|
|
@@ -636,6 +803,7 @@ class AsyncIntel():
|
|
|
636
803
|
|
|
637
804
|
elif result_obj.action_type == "pass":
|
|
638
805
|
pass
|
|
806
|
+
|
|
639
807
|
else:
|
|
640
808
|
raise
|
|
641
809
|
|
|
@@ -647,7 +815,6 @@ class AsyncIntel():
|
|
|
647
815
|
prompt_id: str,
|
|
648
816
|
ExtraFormats: list[object] = [],
|
|
649
817
|
version: str = None,
|
|
650
|
-
inference_save_case = True,
|
|
651
818
|
ConTent_Function = None,
|
|
652
819
|
AConTent_Function = None,
|
|
653
820
|
logger = None,
|
|
@@ -680,7 +847,6 @@ class AsyncIntel():
|
|
|
680
847
|
output_format=output_format,
|
|
681
848
|
prompt_id=prompt_id,
|
|
682
849
|
version=version,
|
|
683
|
-
inference_save_case=inference_save_case,
|
|
684
850
|
)
|
|
685
851
|
if OutputFormat:
|
|
686
852
|
try:
|
|
@@ -722,7 +888,6 @@ class AsyncIntel():
|
|
|
722
888
|
prompt_id: str,
|
|
723
889
|
ExtraFormats: list[object] = [],
|
|
724
890
|
version: str = None,
|
|
725
|
-
inference_save_case = True,
|
|
726
891
|
**kwargs,
|
|
727
892
|
):
|
|
728
893
|
|
|
@@ -742,7 +907,6 @@ class AsyncIntel():
|
|
|
742
907
|
OutputFormat = OutputFormat,
|
|
743
908
|
ExtraFormats = ExtraFormats,
|
|
744
909
|
version = version,
|
|
745
|
-
inference_save_case = inference_save_case,
|
|
746
910
|
**kwargs,
|
|
747
911
|
)
|
|
748
912
|
)
|
|
@@ -829,7 +993,6 @@ class AsyncIntel():
|
|
|
829
993
|
OutputFormat = OutputFormat,
|
|
830
994
|
ExtraFormats = ExtraFormats,
|
|
831
995
|
version = version,
|
|
832
|
-
inference_save_case = False,
|
|
833
996
|
ConTent_Function = ConTent_Function,
|
|
834
997
|
AConTent_Function = AConTent_Function,
|
|
835
998
|
)
|
|
@@ -941,7 +1104,6 @@ class AsyncIntel():
|
|
|
941
1104
|
OutputFormat = OutputFormat,
|
|
942
1105
|
ExtraFormats = ExtraFormats,
|
|
943
1106
|
version = version,
|
|
944
|
-
inference_save_case = False,
|
|
945
1107
|
ConTent_Function = ConTent_Function,
|
|
946
1108
|
AConTent_Function = AConTent_Function,
|
|
947
1109
|
)
|
|
@@ -167,7 +167,26 @@ def create_router(database_url: str,
|
|
|
167
167
|
status_code=500, detail=f"{e}"
|
|
168
168
|
)
|
|
169
169
|
|
|
170
|
+
@router.post("/update_logs",
|
|
171
|
+
description="回滚现行提示词",
|
|
172
|
+
response_model=PromptResponse)
|
|
173
|
+
async def roll_back22(request: RollBackPromptRequest):
|
|
174
|
+
try:
|
|
175
|
+
async with create_async_session(intels.engine) as session:
|
|
170
176
|
|
|
177
|
+
await intels.save_prompt(
|
|
178
|
+
prompt_id = request.prompt_id,
|
|
179
|
+
new_prompt = result.prompt,
|
|
180
|
+
use_case = result.use_case,
|
|
181
|
+
action_type = "inference",
|
|
182
|
+
demand = "",
|
|
183
|
+
score = 61,
|
|
184
|
+
session = session)
|
|
185
|
+
return PromptResponse(msg = "success",content="")
|
|
186
|
+
except Exception as e:
|
|
187
|
+
raise HTTPException(
|
|
188
|
+
status_code=500, detail=f"{e}"
|
|
189
|
+
)
|
|
171
190
|
#系统级别服务
|
|
172
191
|
|
|
173
192
|
@router.post("/sync_database")
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
from dotenv import load_dotenv, find_dotenv
|
|
5
|
-
|
|
6
|
-
dotenv_path = find_dotenv()
|
|
7
|
-
load_dotenv(".env", override=True)
|
|
8
|
-
|
|
9
|
-
from .log import Log
|
|
10
|
-
import logging
|
|
11
|
-
Log_ = Log(console_level = logging.WARNING, # 显示控制台的等级
|
|
12
|
-
log_file_name="app.log")
|
|
13
|
-
logger = Log_.logger
|
|
14
|
-
Log_.set_super_log(logger.critical) # 控制superlog 打印的等级 默认是最高级单独存储一个文件
|
|
15
|
-
super_log = Log_.super_log # 调试工具
|
|
16
|
-
inference_save_case = False
|
|
17
|
-
|
|
18
|
-
from .prompt_craft import AsyncIntel, Intel, IntelNew
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
import os
|
|
3
|
-
from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
|
|
4
|
-
|
|
5
|
-
class Log:
|
|
6
|
-
_instance = None
|
|
7
|
-
def __new__(cls, *args, **kwargs):
|
|
8
|
-
if cls._instance is None:
|
|
9
|
-
cls._instance = super().__new__(cls)
|
|
10
|
-
return cls._instance
|
|
11
|
-
|
|
12
|
-
def __init__(self, console_level = logging.INFO, log_file_name="app.log"):
|
|
13
|
-
self.Console_LOG_LEVEL = console_level
|
|
14
|
-
self.log_file_name = log_file_name
|
|
15
|
-
os.makedirs("logs", exist_ok=True)
|
|
16
|
-
self.LOG_FILE_PATH = os.path.join("logs", log_file_name)
|
|
17
|
-
self.logger = self.get_logger()
|
|
18
|
-
self.super_log_level = self.logger.critical
|
|
19
|
-
|
|
20
|
-
def get_logger(self):
|
|
21
|
-
logger = logging.getLogger()
|
|
22
|
-
# logger.setLevel(self.LOG_LEVEL)
|
|
23
|
-
if not logger.handlers:
|
|
24
|
-
# --- 4. 配置 Formatter (格式化器) ---
|
|
25
|
-
formatter = logging.Formatter(
|
|
26
|
-
"%(asctime)s - %(levelname)s - %(name)s - %(funcName)s - %(module)s:%(lineno)d - %(message)s"
|
|
27
|
-
)
|
|
28
|
-
# --- 5. 配置 Handler (处理器) ---
|
|
29
|
-
|
|
30
|
-
# 5.1 控制台处理器 (StreamHandler)
|
|
31
|
-
console_handler = logging.StreamHandler()
|
|
32
|
-
console_handler.setLevel(self.Console_LOG_LEVEL) # 控制台只显示 INFO 及以上级别的日志
|
|
33
|
-
console_handler.setFormatter(formatter)
|
|
34
|
-
logger.addHandler(console_handler)
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
# 文件
|
|
38
|
-
|
|
39
|
-
# 5.2 信息数据
|
|
40
|
-
|
|
41
|
-
# RotatingFileHandler: 按文件大小轮转
|
|
42
|
-
# maxBytes: 单个日志文件的最大字节数 (例如 10MB)
|
|
43
|
-
# backupCount: 保留的旧日志文件数量
|
|
44
|
-
file_handler = RotatingFileHandler(
|
|
45
|
-
self.LOG_FILE_PATH,
|
|
46
|
-
maxBytes=10 * 1024 * 1024, # 10 MB
|
|
47
|
-
backupCount=5,
|
|
48
|
-
encoding="utf-8",
|
|
49
|
-
)
|
|
50
|
-
file_handler.setLevel(logging.INFO) # 文件中显示所有指定级别的日志
|
|
51
|
-
file_handler.setFormatter(formatter)
|
|
52
|
-
logger.addHandler(file_handler)
|
|
53
|
-
|
|
54
|
-
# 5.3 错误警告日志
|
|
55
|
-
|
|
56
|
-
file_handler_debug = RotatingFileHandler(
|
|
57
|
-
self.LOG_FILE_PATH.replace('.log','_err.log'),
|
|
58
|
-
maxBytes=10 * 1024 * 1024, # 10 MB
|
|
59
|
-
backupCount=5,
|
|
60
|
-
encoding="utf-8",
|
|
61
|
-
)
|
|
62
|
-
file_handler_debug.setLevel(logging.WARNING) # 文件中显示所有指定级别的日志
|
|
63
|
-
file_handler_debug.setFormatter(formatter)
|
|
64
|
-
|
|
65
|
-
file_handler_cri = RotatingFileHandler(
|
|
66
|
-
self.LOG_FILE_PATH.replace('.log','_slog.log'),
|
|
67
|
-
maxBytes=5 * 1024 * 1024, # 10 MB
|
|
68
|
-
backupCount=5,
|
|
69
|
-
encoding="utf-8",
|
|
70
|
-
)
|
|
71
|
-
file_handler_cri.setLevel(logging.CRITICAL) # 文件中显示所有指定级别的日志
|
|
72
|
-
file_handler_cri.setFormatter(formatter)
|
|
73
|
-
logger.addHandler(file_handler_cri)
|
|
74
|
-
return logger
|
|
75
|
-
|
|
76
|
-
def set_super_log(self,logger_info):
|
|
77
|
-
self.super_log_level = logger_info
|
|
78
|
-
|
|
79
|
-
def super_log(self,s, target: str = "target",logger = None):
|
|
80
|
-
logger = logger or self.super_log_level
|
|
81
|
-
logger("\n" + "=="*25 + target +"=="*25 + f"\n type: {str(type(s))}" + f"\ncontent: {s}")
|
|
82
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|