pro-craft 0.1.56__py3-none-any.whl → 0.2.58__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pro-craft might be problematic. Click here for more details.
- pro_craft/__init__.py +2 -10
- pro_craft/code_helper/__init__.py +0 -0
- pro_craft/code_helper/agent.py +90 -0
- pro_craft/code_helper/codermanager.py +143 -0
- pro_craft/code_helper/database.py +36 -0
- pro_craft/code_helper/paper_program.py +183 -0
- pro_craft/code_helper/template_extract.py +134 -0
- pro_craft/code_helper/tools.py +113 -0
- pro_craft/code_helper/vectorstore.py +81 -0
- pro_craft/code_helper/write_code.py +61 -0
- pro_craft/database.py +30 -90
- pro_craft/log.py +63 -33
- pro_craft/prompt_craft/async_.py +190 -401
- pro_craft/server/router/prompt.py +19 -0
- pro_craft/utils.py +1 -1
- {pro_craft-0.1.56.dist-info → pro_craft-0.2.58.dist-info}/METADATA +7 -1
- pro_craft-0.2.58.dist-info/RECORD +28 -0
- pro_craft/code_helper/coder.py +0 -660
- pro_craft/code_helper/designer.py +0 -115
- pro_craft-0.1.56.dist-info/RECORD +0 -21
- {pro_craft-0.1.56.dist-info → pro_craft-0.2.58.dist-info}/WHEEL +0 -0
- {pro_craft-0.1.56.dist-info → pro_craft-0.2.58.dist-info}/top_level.txt +0 -0
pro_craft/prompt_craft/async_.py
CHANGED
|
@@ -1,39 +1,37 @@
|
|
|
1
1
|
# 测试1
|
|
2
2
|
from pro_craft.utils import extract_
|
|
3
|
-
from pro_craft import logger as pro_craft_logger
|
|
4
3
|
from modusched.core import BianXieAdapter, ArkAdapter
|
|
5
|
-
from datetime import datetime
|
|
6
4
|
from enum import Enum
|
|
7
5
|
import functools
|
|
8
6
|
import json
|
|
9
7
|
import os
|
|
10
|
-
from pro_craft.database import Prompt, UseCase, PromptBase
|
|
8
|
+
from pro_craft.database import Prompt, UseCase, PromptBase, SyncMetadata
|
|
11
9
|
from pro_craft.utils import create_session, create_async_session
|
|
12
10
|
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine # 异步核心
|
|
13
11
|
from sqlalchemy import select, delete # 导入 select, delete 用于异步操作
|
|
14
12
|
import inspect
|
|
15
|
-
from datetime import datetime
|
|
16
13
|
from pro_craft.utils import extract_
|
|
17
14
|
import asyncio
|
|
18
15
|
import re
|
|
19
16
|
from pydantic import BaseModel, ValidationError, field_validator
|
|
20
17
|
from sqlalchemy import select, desc
|
|
21
18
|
from json.decoder import JSONDecodeError
|
|
22
|
-
|
|
23
|
-
from datetime import datetime, timedelta
|
|
24
|
-
from datetime import datetime, timedelta
|
|
19
|
+
|
|
25
20
|
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
|
26
21
|
from sqlalchemy import select, and_ # 引入 select 和 and_
|
|
27
22
|
from sqlalchemy.orm import class_mapper # 用于检查对象是否是持久化的
|
|
28
23
|
from datetime import datetime, timedelta
|
|
29
|
-
|
|
24
|
+
import datetime
|
|
30
25
|
from tqdm.asyncio import tqdm
|
|
31
26
|
import pandas as pd
|
|
32
27
|
import plotly.graph_objects as go
|
|
33
|
-
from pro_craft import super_log
|
|
34
28
|
|
|
35
|
-
|
|
36
|
-
|
|
29
|
+
def get_log_info(target: str, val = None):
|
|
30
|
+
return f"{target} & {type(val)} & {val}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
BATCH_SIZE = int(os.getenv("DATABASE_SYNC_BATCH_SIZE",1000))
|
|
37
35
|
|
|
38
36
|
def fix_broken_json_string(broken_json_str):
|
|
39
37
|
# 移除 BOM
|
|
@@ -114,12 +112,11 @@ class AsyncIntel():
|
|
|
114
112
|
logger = None,
|
|
115
113
|
):
|
|
116
114
|
database_url = database_url or os.getenv("database_url")
|
|
117
|
-
self.logger = logger
|
|
115
|
+
self.logger = logger
|
|
118
116
|
try:
|
|
119
117
|
assert database_url
|
|
120
118
|
assert 'aio' in database_url
|
|
121
119
|
except AssertionError as e:
|
|
122
|
-
super_log(database_url,'database_url',logger=self.logger.warning)
|
|
123
120
|
raise IntellectRemoveFormatError(f"异步服务url必须提供, 且必须是aiomysql配置") from e
|
|
124
121
|
|
|
125
122
|
self.engine = create_async_engine(database_url, echo=False,
|
|
@@ -159,29 +156,6 @@ class AsyncIntel():
|
|
|
159
156
|
async with engine.begin() as conn:
|
|
160
157
|
await conn.run_sync(PromptBase.metadata.create_all)
|
|
161
158
|
|
|
162
|
-
async def get_prompt(self,prompt_id,version,session):
|
|
163
|
-
"""
|
|
164
|
-
获取指定 prompt_id 的最新版本数据,通过创建时间判断。
|
|
165
|
-
"""
|
|
166
|
-
if version:
|
|
167
|
-
stmt_ = select(Prompt).filter(
|
|
168
|
-
Prompt.prompt_id == prompt_id,
|
|
169
|
-
Prompt.version == version
|
|
170
|
-
)
|
|
171
|
-
else:
|
|
172
|
-
stmt_ = select(Prompt).filter(
|
|
173
|
-
Prompt.prompt_id == prompt_id,
|
|
174
|
-
)
|
|
175
|
-
stmt = stmt_.order_by(
|
|
176
|
-
desc(Prompt.timestamp), # 使用 sqlalchemy.desc() 来指定降序
|
|
177
|
-
desc(Prompt.version) # 使用 sqlalchemy.desc() 来指定降序
|
|
178
|
-
)
|
|
179
|
-
|
|
180
|
-
result = await session.execute(stmt)
|
|
181
|
-
result = result.scalars().first()
|
|
182
|
-
|
|
183
|
-
return result
|
|
184
|
-
|
|
185
159
|
async def sync_production_database(self,database_url:str):
|
|
186
160
|
target_engine = create_async_engine(database_url, echo=False)
|
|
187
161
|
await self.create_database(target_engine)
|
|
@@ -275,7 +249,29 @@ class AsyncIntel():
|
|
|
275
249
|
else:
|
|
276
250
|
print("No new records to sync.")
|
|
277
251
|
|
|
252
|
+
async def get_prompt(self,prompt_id,version,session):
|
|
253
|
+
"""
|
|
254
|
+
获取指定 prompt_id 的最新版本数据,通过创建时间判断。
|
|
255
|
+
"""
|
|
256
|
+
if version:
|
|
257
|
+
stmt_ = select(Prompt).filter(
|
|
258
|
+
Prompt.prompt_id == prompt_id,
|
|
259
|
+
Prompt.version == version
|
|
260
|
+
)
|
|
261
|
+
else:
|
|
262
|
+
stmt_ = select(Prompt).filter(
|
|
263
|
+
Prompt.prompt_id == prompt_id,
|
|
264
|
+
)
|
|
265
|
+
stmt = stmt_.order_by(
|
|
266
|
+
desc(Prompt.timestamp), # 使用 sqlalchemy.desc() 来指定降序
|
|
267
|
+
desc(Prompt.version) # 使用 sqlalchemy.desc() 来指定降序
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
result = await session.execute(stmt)
|
|
271
|
+
result = result.scalars().first()
|
|
278
272
|
|
|
273
|
+
return result
|
|
274
|
+
|
|
279
275
|
async def get_prompt_safe(self,
|
|
280
276
|
prompt_id: str,
|
|
281
277
|
version = None,
|
|
@@ -286,12 +282,13 @@ class AsyncIntel():
|
|
|
286
282
|
prompt_obj = await self.get_prompt(prompt_id=prompt_id,version=version,session=session)
|
|
287
283
|
if prompt_obj:
|
|
288
284
|
return prompt_obj
|
|
285
|
+
if version:
|
|
286
|
+
prompt_obj = await self.get_prompt(prompt_id=prompt_id,version=None,session=session)
|
|
289
287
|
|
|
290
|
-
|
|
291
|
-
|
|
288
|
+
if prompt_obj is None:
|
|
289
|
+
raise IntellectRemoveError("不存在的prompt_id")
|
|
292
290
|
return prompt_obj
|
|
293
291
|
|
|
294
|
-
|
|
295
292
|
async def save_prompt(self,
|
|
296
293
|
prompt_id: str,
|
|
297
294
|
new_prompt: str,
|
|
@@ -332,204 +329,16 @@ class AsyncIntel():
|
|
|
332
329
|
session.add(prompt1)
|
|
333
330
|
await session.commit() # 提交事务,将数据写入数据库
|
|
334
331
|
|
|
335
|
-
async def
|
|
336
|
-
target_prompt_id: str,
|
|
337
|
-
start_time: datetime = None, # 新增:开始时间
|
|
338
|
-
end_time: datetime = None, # 新增:结束时间
|
|
339
|
-
session = None
|
|
340
|
-
):
|
|
341
|
-
"""
|
|
342
|
-
从sql保存提示词
|
|
343
|
-
"""
|
|
344
|
-
stmt = select(UseCase).filter(UseCase.is_deleted == 0,
|
|
345
|
-
UseCase.prompt_id == target_prompt_id)
|
|
346
|
-
|
|
347
|
-
if start_time:
|
|
348
|
-
stmt = stmt.filter(UseCase.created_at >= start_time) # 假设你的UseCase模型有一个created_at字段
|
|
349
|
-
|
|
350
|
-
if end_time:
|
|
351
|
-
stmt = stmt.filter(UseCase.created_at <= end_time)
|
|
352
|
-
result = await session.execute(stmt)
|
|
353
|
-
# use_case = result.scalars().one_or_none()
|
|
354
|
-
use_case = result.scalars().all()
|
|
355
|
-
return use_case
|
|
356
|
-
|
|
357
|
-
async def save_use_case(self,
|
|
358
|
-
prompt_id: str,
|
|
359
|
-
use_case:str = "",
|
|
360
|
-
timestamp = "",
|
|
361
|
-
output = "",
|
|
362
|
-
solution: str = "",
|
|
363
|
-
faired_time = 0,
|
|
364
|
-
session = None
|
|
365
|
-
):
|
|
366
|
-
|
|
367
|
-
"""
|
|
368
|
-
从sql保存提示词
|
|
369
|
-
"""
|
|
370
|
-
#TODO 存之前保证数据库中相同的prompt_id中没有重复的use_case
|
|
371
|
-
use_cases = await self.get_use_case(target_prompt_id = prompt_id,
|
|
372
|
-
session = session)
|
|
373
|
-
for use_case_old in use_cases:
|
|
374
|
-
if use_case == use_case_old.use_case:
|
|
375
|
-
# print("用例已经存在")
|
|
376
|
-
return
|
|
377
|
-
|
|
378
|
-
use_case = UseCase(prompt_id=prompt_id,
|
|
379
|
-
use_case = use_case,
|
|
380
|
-
timestamp = timestamp,
|
|
381
|
-
output = output,
|
|
382
|
-
solution = solution,
|
|
383
|
-
faired_time = faired_time,
|
|
384
|
-
)
|
|
385
|
-
|
|
386
|
-
session.add(use_case)
|
|
387
|
-
await session.commit() # 提交事务,将数据写入数据库
|
|
388
|
-
|
|
389
|
-
async def push_action_order(self,demand : str,prompt_id: str,
|
|
390
|
-
action_type = 'train'):# init
|
|
391
|
-
|
|
392
|
-
"""
|
|
393
|
-
从sql保存提示词
|
|
394
|
-
推一个train 状态到指定的位置
|
|
395
|
-
|
|
396
|
-
将打算修改的状态推上数据库 # 1
|
|
397
|
-
"""
|
|
398
|
-
# 查看是否已经存在
|
|
399
|
-
async with create_async_session(self.engine) as session:
|
|
332
|
+
async def adjust_prompt(self,prompt_id: str,action_type = "summary", demand: str = ""):
|
|
400
333
|
|
|
401
|
-
latest_prompt = await self.get_prompt_safe(prompt_id=prompt_id,session=session)
|
|
402
|
-
if latest_prompt:
|
|
403
|
-
await self.save_prompt(prompt_id=latest_prompt.prompt_id,
|
|
404
|
-
new_prompt = latest_prompt.prompt,
|
|
405
|
-
use_case = latest_prompt.use_case,
|
|
406
|
-
action_type=action_type,
|
|
407
|
-
demand=demand,
|
|
408
|
-
score=latest_prompt.score,
|
|
409
|
-
session=session
|
|
410
|
-
)
|
|
411
|
-
return "success"
|
|
412
|
-
else:
|
|
413
|
-
await self.save_prompt(prompt_id=prompt_id,
|
|
414
|
-
new_prompt = demand,
|
|
415
|
-
use_case = "",
|
|
416
|
-
action_type="inference",
|
|
417
|
-
demand=demand,
|
|
418
|
-
score=60,
|
|
419
|
-
session=session
|
|
420
|
-
)
|
|
421
|
-
return "init"
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
async def intellect(self,
|
|
425
|
-
input_data: dict | str,
|
|
426
|
-
output_format: str,
|
|
427
|
-
prompt_id: str,
|
|
428
|
-
version: str = None,
|
|
429
|
-
inference_save_case = True,
|
|
430
|
-
change_case = False,
|
|
431
|
-
):
|
|
432
|
-
"""
|
|
433
|
-
自定自动化执行命令的方法,
|
|
434
|
-
不涉及严格的校验, 主要职能在自动化的修改提示词, 或者管理提示词上
|
|
435
|
-
"""
|
|
436
|
-
if isinstance(input_data,dict):
|
|
437
|
-
input_ = json.dumps(input_data,ensure_ascii=False)
|
|
438
|
-
elif isinstance(input_data,str):
|
|
439
|
-
input_ = input_data
|
|
440
|
-
|
|
441
334
|
# 查数据库, 获取最新提示词对象
|
|
442
335
|
async with create_async_session(self.engine) as session:
|
|
443
336
|
result_obj = await self.get_prompt_safe(prompt_id=prompt_id,session=session)
|
|
444
|
-
if result_obj is None:
|
|
445
|
-
raise IntellectRemoveError("不存在的prompt_id")
|
|
446
337
|
|
|
447
338
|
prompt = result_obj.prompt
|
|
448
|
-
|
|
449
|
-
# 直接推理即可
|
|
450
|
-
ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" + input_)
|
|
451
|
-
if inference_save_case:
|
|
452
|
-
# 设计一个机制, 传输说获取300数据, 那么数据就一直流转获取, 知道300截止
|
|
453
|
-
await self.save_use_case(prompt_id,
|
|
454
|
-
use_case = input_,
|
|
455
|
-
timestamp = datetime.now(),
|
|
456
|
-
output = ai_result,
|
|
457
|
-
solution = output_format,
|
|
458
|
-
faired_time = 0,
|
|
459
|
-
session = session,
|
|
460
|
-
)
|
|
461
|
-
|
|
462
|
-
elif result_obj.action_type == "train":
|
|
463
|
-
assert result_obj.demand # 如果type = train 且 demand 是空 则报错
|
|
464
|
-
# 则训练推广
|
|
465
|
-
|
|
466
|
-
# 新版本 默人修改会 inference 状态
|
|
467
|
-
|
|
339
|
+
use_case = result_obj.use_case
|
|
468
340
|
|
|
469
|
-
|
|
470
|
-
# # 注意, 这里的调整要求使用最初的那个输入, 最好一口气调整好
|
|
471
|
-
# chat_history = prompt
|
|
472
|
-
# if input_ == before_input: # 输入没变, 说明还是针对同一个输入进行讨论
|
|
473
|
-
# # input_prompt = chat_history + "\nuser:" + demand
|
|
474
|
-
# input_prompt = chat_history + "\nuser:" + demand + output_format
|
|
475
|
-
# else:
|
|
476
|
-
# # input_prompt = chat_history + "\nuser:" + demand + "\n-----input----\n" + input_
|
|
477
|
-
# input_prompt = chat_history + "\nuser:" + demand + output_format + "\n-----input----\n" + input_
|
|
478
|
-
|
|
479
|
-
# ai_result = await self.llm.aproduct(input_prompt)
|
|
480
|
-
# chat_history = input_prompt + "\nassistant:\n" + ai_result # 用聊天记录作为完整提示词
|
|
481
|
-
# await self.save_prompt(prompt_id, chat_history,
|
|
482
|
-
# use_case = input_,
|
|
483
|
-
# score = 60,
|
|
484
|
-
# session = session)
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
# version 2
|
|
488
|
-
|
|
489
|
-
# if input_ == before_input:
|
|
490
|
-
# new_prompt = prompt + "\nuser:" + demand
|
|
491
|
-
# else:
|
|
492
|
-
# new_prompt = prompt + "\nuser:" + input_
|
|
493
|
-
|
|
494
|
-
# ai_result = await self.llm.aproduct(new_prompt + output_format)
|
|
495
|
-
|
|
496
|
-
# save_new_prompt = new_prompt + "\nassistant:\n" + ai_result
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
# await self.save_prompt(
|
|
500
|
-
# prompt_id,
|
|
501
|
-
# new_prompt=save_new_prompt,
|
|
502
|
-
# use_case = input_,
|
|
503
|
-
# action_type = "inference",
|
|
504
|
-
# score = 60,
|
|
505
|
-
# session = session)
|
|
506
|
-
chat_history = prompt
|
|
507
|
-
before_input = result_obj.use_case
|
|
508
|
-
demand = result_obj.demand
|
|
509
|
-
input_data = input_
|
|
510
|
-
if before_input == "" or change_case is True:
|
|
511
|
-
result_obj.use_case = input_
|
|
512
|
-
await session.commit()
|
|
513
|
-
# 查询上一条, 将before_input 更新位input_
|
|
514
|
-
prompt += input_
|
|
515
|
-
|
|
516
|
-
# 使用更新后的数据进行后续步骤
|
|
517
|
-
new_prompt = prompt + "\nuser:" + demand
|
|
518
|
-
|
|
519
|
-
ai_result = await self.llm.aproduct(new_prompt + output_format)
|
|
520
|
-
|
|
521
|
-
save_new_prompt = new_prompt + "\nassistant:\n" + ai_result
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
await self.save_prompt(
|
|
525
|
-
prompt_id,
|
|
526
|
-
new_prompt=save_new_prompt,
|
|
527
|
-
use_case = input_,
|
|
528
|
-
action_type = "inference",
|
|
529
|
-
score = 60,
|
|
530
|
-
session = session)
|
|
531
|
-
|
|
532
|
-
elif result_obj.action_type == "summary":
|
|
341
|
+
if action_type == "summary":
|
|
533
342
|
system_prompt_summary = """
|
|
534
343
|
很棒, 我们已经达成了某种默契, 我们之间合作无间, 但是, 可悲的是, 当我关闭这个窗口的时候, 你就会忘记我们之间经历的种种磨合, 这是可惜且心痛的, 所以你能否将目前这一套处理流程结晶成一个优质的prompt 这样, 我们下一次只要将prompt输入, 你就能想起我们今天的磨合过程,
|
|
535
344
|
对了,我提示一点, 这个prompt的主角是你, 也就是说, 你在和未来的你对话, 你要教会未来的你今天这件事, 是否让我看懂到时其次
|
|
@@ -537,24 +346,12 @@ class AsyncIntel():
|
|
|
537
346
|
只要输出提示词内容即可, 不需要任何的说明和解释
|
|
538
347
|
"""
|
|
539
348
|
|
|
540
|
-
latest_prompt = await self.get_prompt_safe(prompt_id=prompt_id,session=session)
|
|
541
|
-
|
|
542
349
|
system_result = await self.llm.aproduct(prompt + system_prompt_summary)
|
|
543
350
|
s_prompt = extract_(system_result,pattern_key=r"prompt")
|
|
544
351
|
new_prompt = s_prompt or system_result
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
use_case = latest_prompt.use_case,
|
|
549
|
-
score = 65,
|
|
550
|
-
action_type = "inference",
|
|
551
|
-
session = session
|
|
552
|
-
)
|
|
553
|
-
|
|
554
|
-
ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" + input_)
|
|
555
|
-
|
|
556
|
-
elif result_obj.action_type == "finetune":
|
|
557
|
-
demand = result_obj.demand
|
|
352
|
+
|
|
353
|
+
elif action_type == "finetune":
|
|
354
|
+
assert demand
|
|
558
355
|
change_by_opinion_prompt = """
|
|
559
356
|
你是一个资深AI提示词工程师,具备卓越的Prompt设计与优化能力。
|
|
560
357
|
我将为你提供一段现有System Prompt。你的核心任务是基于这段Prompt进行修改,以实现我提出的特定目标和功能需求。
|
|
@@ -581,76 +378,45 @@ class AsyncIntel():
|
|
|
581
378
|
功能需求:
|
|
582
379
|
{opinion}
|
|
583
380
|
"""
|
|
381
|
+
new_prompt = await self.llm.aproduct(
|
|
382
|
+
change_by_opinion_prompt.format(old_system_prompt=prompt, opinion=demand)
|
|
383
|
+
)
|
|
584
384
|
|
|
585
|
-
|
|
586
|
-
prompt_ = await self.get_prompt_safe(prompt_id = prompt_id,version = version,
|
|
587
|
-
session=session)
|
|
385
|
+
elif action_type == "patch":
|
|
588
386
|
assert demand
|
|
589
|
-
|
|
590
|
-
if demand:
|
|
591
|
-
new_prompt = await self.llm.aproduct(
|
|
592
|
-
change_by_opinion_prompt.format(old_system_prompt=prompt_.prompt, opinion=demand)
|
|
593
|
-
)
|
|
594
|
-
else:
|
|
595
|
-
new_prompt = prompt_
|
|
596
|
-
await self.save_prompt(
|
|
597
|
-
prompt_id,
|
|
598
|
-
new_prompt = new_prompt,
|
|
599
|
-
use_case = latest_prompt.use_case,
|
|
600
|
-
score = 70,
|
|
601
|
-
action_type = "inference",
|
|
602
|
-
session = session
|
|
603
|
-
)
|
|
604
|
-
|
|
605
|
-
ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" + input_)
|
|
387
|
+
new_prompt = prompt + "\n"+demand,
|
|
606
388
|
|
|
607
|
-
elif
|
|
608
|
-
demand = result_obj.demand
|
|
609
|
-
assert demand
|
|
610
|
-
latest_prompt = await self.get_prompt_safe(prompt_id=prompt_id,session=session)
|
|
611
|
-
|
|
612
|
-
chat_history = prompt + demand
|
|
613
|
-
await self.save_prompt(prompt_id,
|
|
614
|
-
chat_history,
|
|
615
|
-
use_case = latest_prompt.use_case,
|
|
616
|
-
score = 70,
|
|
617
|
-
action_type = "inference",
|
|
618
|
-
session = session)
|
|
619
|
-
|
|
620
|
-
ai_result = await self.llm.aproduct(chat_history + output_format + "\nuser:" + input_)
|
|
621
|
-
|
|
622
|
-
elif result_obj.action_type.startswith("to:"):
|
|
389
|
+
elif action_type.startswith("to:"):
|
|
623
390
|
target_version = result_obj.action_type.split(":")[-1]
|
|
624
|
-
latest_prompt = await self.get_prompt_safe(prompt_id=prompt_id,session=session)
|
|
625
391
|
prompt_obj = await self.get_prompt_safe(prompt_id=prompt_id,
|
|
626
392
|
version=target_version,
|
|
627
393
|
session=session)
|
|
628
394
|
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
use_case = latest_prompt.use_case,
|
|
632
|
-
score = prompt_obj.score,
|
|
633
|
-
action_type = "inference",
|
|
634
|
-
session = session)
|
|
635
|
-
ai_result = await self.llm.aproduct(prompt_obj.prompt + output_format + "\nuser:" + input_)
|
|
636
|
-
|
|
637
|
-
elif result_obj.action_type == "pass":
|
|
638
|
-
pass
|
|
395
|
+
new_prompt = prompt_obj.prompt
|
|
396
|
+
|
|
639
397
|
else:
|
|
640
398
|
raise
|
|
641
399
|
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
400
|
+
await self.save_prompt(
|
|
401
|
+
prompt_id,
|
|
402
|
+
new_prompt = new_prompt,
|
|
403
|
+
use_case = use_case,
|
|
404
|
+
score = 70,
|
|
405
|
+
action_type = "inference",
|
|
406
|
+
session = session
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
return "success"
|
|
410
|
+
|
|
411
|
+
async def inference_format(self,
|
|
645
412
|
input_data: dict | str,
|
|
646
|
-
OutputFormat: object | None,
|
|
647
413
|
prompt_id: str,
|
|
648
|
-
ExtraFormats: list[object] = [],
|
|
649
414
|
version: str = None,
|
|
650
|
-
|
|
415
|
+
OutputFormat: object | None = None,
|
|
416
|
+
ExtraFormats: list[object] = [],
|
|
651
417
|
ConTent_Function = None,
|
|
652
418
|
AConTent_Function = None,
|
|
653
|
-
|
|
419
|
+
again = True,
|
|
654
420
|
):
|
|
655
421
|
"""
|
|
656
422
|
这个format 是严格校验模式, 是interllect 的增强版, 会主动校验内容,并及时抛出异常(或者伺机修正)
|
|
@@ -667,137 +433,175 @@ class AsyncIntel():
|
|
|
667
433
|
"```json([\s\S]*?)```"
|
|
668
434
|
使用以下方式验证
|
|
669
435
|
"""
|
|
670
|
-
|
|
671
|
-
output_format = base_format_prompt + "\n".join([inspect.getsource(outputformat) for outputformat in ExtraFormats]) + inspect.getsource(OutputFormat)
|
|
672
|
-
else:
|
|
673
|
-
output_format = ""
|
|
436
|
+
assert isinstance(input_data,(dict,str))
|
|
674
437
|
|
|
675
|
-
if
|
|
676
|
-
|
|
438
|
+
input_ = json.dumps(input_data,ensure_ascii=False) if isinstance(input_data,dict) else input_data
|
|
439
|
+
output_format = base_format_prompt + "\n".join([inspect.getsource(outputformat) for outputformat in ExtraFormats]) + inspect.getsource(OutputFormat) if OutputFormat else ""
|
|
440
|
+
self.logger and self.logger.info(get_log_info("intel-输入",input_data))
|
|
677
441
|
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
if OutputFormat:
|
|
442
|
+
async with create_async_session(self.engine) as session:
|
|
443
|
+
result_obj = await self.get_prompt_safe(prompt_id=prompt_id,version= version,
|
|
444
|
+
session=session)
|
|
445
|
+
prompt = result_obj.prompt
|
|
446
|
+
ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" + input_)
|
|
447
|
+
|
|
448
|
+
def check_json_valid(ai_result,OutputFormat):
|
|
686
449
|
try:
|
|
687
450
|
json_str = extract_(ai_result,r'json')
|
|
688
451
|
ai_result = json.loads(json_str)
|
|
689
452
|
OutputFormat(**ai_result)
|
|
690
453
|
|
|
691
454
|
except JSONDecodeError as e:
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
ai_result = json.loads(json_str)
|
|
696
|
-
OutputFormat(**ai_result)
|
|
697
|
-
|
|
698
|
-
except JSONDecodeError as e:
|
|
699
|
-
self.logger.error(f'{type(json_str)} $ {prompt_id}intellect生成的内容为无法被Json解析 $ {json_str}')
|
|
700
|
-
raise IntellectRemoveFormatError(f"prompt_id: {prompt_id} 生成的内容为无法被Json解析 {e}") from e
|
|
701
|
-
|
|
455
|
+
self.logger.error(f'{type(json_str)} $ {prompt_id}intellect生成的内容为无法被Json解析 $ {json_str}')
|
|
456
|
+
# raise IntellectRemoveFormatError(f"prompt_id: {prompt_id} 生成的内容为无法被Json解析 {e}") from e
|
|
457
|
+
return 0
|
|
702
458
|
except ValidationError as e:
|
|
703
459
|
err_info = e.errors()[0]
|
|
704
|
-
|
|
705
|
-
|
|
460
|
+
self.logger.error(f'{type(json_str)} $ {prompt_id}解析未通过OutputFormat $ {json_str}')
|
|
461
|
+
# raise IntellectRemoveFormatError(f"{err_info["type"]}: 属性:{err_info['loc']}, 发生了如下错误: {err_info['msg']}, 格式校验失败, 当前输入为: {err_info['input']} 请检查") from e
|
|
462
|
+
return 0
|
|
706
463
|
except Exception as e:
|
|
707
464
|
raise Exception(f"Error {prompt_id} : {e}") from e
|
|
465
|
+
return 1
|
|
708
466
|
|
|
709
|
-
if
|
|
467
|
+
if OutputFormat:
|
|
468
|
+
check_result = check_json_valid(ai_result,OutputFormat)
|
|
469
|
+
if check_result ==0 and again:
|
|
470
|
+
ai_result = await self.llm.aproduct(ai_result + output_format)
|
|
471
|
+
check_result_ = check_json_valid(ai_result,OutputFormat)
|
|
472
|
+
if check_result_ ==0:
|
|
473
|
+
raise IntellectRemoveFormatError(f"prompt_id: {prompt_id} 多次生成的内容均未通过OutputFormat校验, 当前内容为: {ai_result}")
|
|
474
|
+
json_str = extract_(ai_result,r'json')
|
|
475
|
+
ai_result = json.loads(json_str)
|
|
476
|
+
|
|
477
|
+
if ConTent_Function:# TODO
|
|
710
478
|
ConTent_Function(ai_result,input_data)
|
|
711
479
|
|
|
712
480
|
if AConTent_Function:
|
|
713
481
|
await AConTent_Function(ai_result,input_data)
|
|
714
482
|
|
|
715
|
-
|
|
716
|
-
logger.info(f'{type(ai_result)} $ intellect输出 ai_result $ {ai_result}')
|
|
483
|
+
self.logger and self.logger.info(f'{type(ai_result)} $ intellect输出 ai_result $ {ai_result}')
|
|
717
484
|
return ai_result
|
|
718
485
|
|
|
719
|
-
async def
|
|
486
|
+
async def inference_format_gather(self,
|
|
720
487
|
input_datas: list[dict | str],
|
|
721
|
-
OutputFormat: object | None,
|
|
722
488
|
prompt_id: str,
|
|
723
|
-
ExtraFormats: list[object] = [],
|
|
724
489
|
version: str = None,
|
|
725
|
-
|
|
490
|
+
OutputFormat: object | None = None,
|
|
491
|
+
ExtraFormats: list[object] = [],
|
|
726
492
|
**kwargs,
|
|
727
493
|
):
|
|
728
494
|
|
|
729
|
-
async with create_async_session(self.engine) as session:
|
|
730
|
-
prompt_result = await self.get_prompt_safe(prompt_id=prompt_id,
|
|
731
|
-
session=session)
|
|
732
|
-
if prompt_result is None:
|
|
733
|
-
raise IntellectRemoveError("不存在的prompt_id")
|
|
734
|
-
if prompt_result.action_type != "inference":
|
|
735
|
-
input_datas = input_datas[:1]
|
|
736
495
|
tasks = []
|
|
737
496
|
for input_data in input_datas:
|
|
738
497
|
tasks.append(
|
|
739
|
-
self.
|
|
498
|
+
self.inference_format(
|
|
740
499
|
input_data = input_data,
|
|
741
500
|
prompt_id = prompt_id,
|
|
501
|
+
version = version,
|
|
742
502
|
OutputFormat = OutputFormat,
|
|
743
503
|
ExtraFormats = ExtraFormats,
|
|
744
|
-
version = version,
|
|
745
|
-
inference_save_case = inference_save_case,
|
|
746
504
|
**kwargs,
|
|
747
505
|
)
|
|
748
506
|
)
|
|
749
|
-
results = await
|
|
507
|
+
results = await tqdm.gather(*tasks,total=len(tasks))
|
|
508
|
+
# results = await asyncio.gather(*tasks, return_exceptions=False)
|
|
750
509
|
return results
|
|
751
510
|
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
async def get_use_case(self,
|
|
515
|
+
target_prompt_id: str,
|
|
516
|
+
start_time: datetime = None, # 新增:开始时间
|
|
517
|
+
end_time: datetime = None, # 新增:结束时间
|
|
518
|
+
session = None
|
|
519
|
+
):
|
|
520
|
+
"""
|
|
521
|
+
从sql保存提示词
|
|
522
|
+
"""
|
|
523
|
+
stmt = select(UseCase).filter(UseCase.is_deleted == 0,
|
|
524
|
+
UseCase.prompt_id == target_prompt_id)
|
|
525
|
+
|
|
526
|
+
if start_time:
|
|
527
|
+
stmt = stmt.filter(UseCase.timestamp >= start_time) # 假设你的UseCase模型有一个created_at字段
|
|
528
|
+
|
|
529
|
+
if end_time:
|
|
530
|
+
stmt = stmt.filter(UseCase.timestamp <= end_time)
|
|
531
|
+
result = await session.execute(stmt)
|
|
532
|
+
# use_case = result.scalars().one_or_none()
|
|
533
|
+
use_case = result.scalars().all()
|
|
534
|
+
return use_case
|
|
535
|
+
|
|
536
|
+
async def save_use_case(self,log_file,session = None):
|
|
537
|
+
with open(log_file,'r') as f:
|
|
538
|
+
x = f.read()
|
|
539
|
+
|
|
540
|
+
def deal_log(resu):
|
|
541
|
+
if len(resu) <3:
|
|
542
|
+
return
|
|
543
|
+
try:
|
|
544
|
+
create_time = resu[1]
|
|
545
|
+
level = resu[2]
|
|
546
|
+
funcname = resu[3]
|
|
547
|
+
line = resu[4]
|
|
548
|
+
pathname = resu[5]
|
|
549
|
+
message = resu[6]
|
|
550
|
+
|
|
551
|
+
message_list = message.split("&")
|
|
552
|
+
if len(message_list) == 3:
|
|
553
|
+
target, type_, content = message_list
|
|
554
|
+
elif len(message_list) == 2:
|
|
555
|
+
target, type_ = message_list
|
|
556
|
+
content = "只有两个"
|
|
557
|
+
elif len(message_list) == 1:
|
|
558
|
+
target = message_list[0]
|
|
559
|
+
type_ = " "
|
|
560
|
+
content = "只有一个"
|
|
561
|
+
|
|
562
|
+
dt_object = datetime.datetime.fromtimestamp(float(create_time.strip()))
|
|
563
|
+
use_case = UseCase(
|
|
564
|
+
time = create_time,
|
|
565
|
+
level = level,
|
|
566
|
+
timestamp =dt_object.strftime('%Y-%m-%d %H:%M:%S.%f'),
|
|
567
|
+
filepath=pathname,
|
|
568
|
+
function=funcname,
|
|
569
|
+
lines=line,
|
|
570
|
+
type_=type_,
|
|
571
|
+
target=target,
|
|
572
|
+
content=content,
|
|
770
573
|
)
|
|
574
|
+
session.add(use_case)
|
|
575
|
+
except Exception as e:
|
|
576
|
+
print(resu,'resu')
|
|
577
|
+
raise
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
for res in x.split("||"):
|
|
581
|
+
resu = res.split("$")
|
|
582
|
+
deal_log(resu)
|
|
583
|
+
|
|
584
|
+
await session.commit() # 提交事务,将数据写入数据库
|
|
771
585
|
|
|
772
|
-
#######
|
|
773
|
-
kwargs.update({"input_data":output_})
|
|
774
|
-
result = await func(*args, **kwargs)
|
|
775
|
-
return result
|
|
776
|
-
return wrapper
|
|
777
|
-
return outer_packing
|
|
778
586
|
|
|
779
587
|
async def intellect_format_eval(self,
|
|
780
|
-
OutputFormat: object,
|
|
781
588
|
prompt_id: str,
|
|
589
|
+
version: str = None,
|
|
782
590
|
database_url = None,
|
|
591
|
+
OutputFormat: object = None,
|
|
783
592
|
ExtraFormats: list[object] = [],
|
|
784
|
-
version: str = None,
|
|
785
593
|
MIN_SUCCESS_RATE = 80.0,
|
|
786
594
|
ConTent_Function = None,
|
|
787
595
|
AConTent_Function = None,
|
|
596
|
+
start = None,
|
|
597
|
+
end = None,
|
|
788
598
|
):
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
# TODO 人类评价 eval
|
|
792
|
-
# TODO llm 评价 eval
|
|
793
|
-
"""
|
|
599
|
+
# start = datetime(2023, 1, 1, 10, 0, 0)
|
|
600
|
+
# end = datetime(2023, 1, 15, 12, 30, 0)
|
|
794
601
|
async with create_async_session(self.engine) as session:
|
|
795
602
|
prompt_result = await self.get_prompt_safe(prompt_id=prompt_id,
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
raise IntellectRemoveError("不存在的prompt_id")
|
|
799
|
-
if prompt_result.action_type != "inference":
|
|
800
|
-
raise IntellectRemoveError("请在inference模式下使用次类")
|
|
603
|
+
version = version,
|
|
604
|
+
session=session)
|
|
801
605
|
|
|
802
606
|
if database_url:
|
|
803
607
|
eval_engine = create_async_engine(database_url, echo=False,
|
|
@@ -809,12 +613,11 @@ class AsyncIntel():
|
|
|
809
613
|
)
|
|
810
614
|
else:
|
|
811
615
|
eval_engine = self.engine
|
|
616
|
+
|
|
812
617
|
async with create_async_session(eval_engine) as eval_session:
|
|
813
|
-
# start = datetime(2023, 1, 1, 10, 0, 0)
|
|
814
|
-
# end = datetime(2023, 1, 15, 12, 30, 0)
|
|
815
618
|
use_cases = await self.get_use_case(target_prompt_id=prompt_id,session=eval_session,
|
|
816
|
-
start_time=
|
|
817
|
-
end_time=
|
|
619
|
+
start_time=start,
|
|
620
|
+
end_time=end,)
|
|
818
621
|
|
|
819
622
|
total_assertions = len(use_cases)
|
|
820
623
|
result_cases = []
|
|
@@ -829,7 +632,6 @@ class AsyncIntel():
|
|
|
829
632
|
OutputFormat = OutputFormat,
|
|
830
633
|
ExtraFormats = ExtraFormats,
|
|
831
634
|
version = version,
|
|
832
|
-
inference_save_case = False,
|
|
833
635
|
ConTent_Function = ConTent_Function,
|
|
834
636
|
AConTent_Function = AConTent_Function,
|
|
835
637
|
)
|
|
@@ -875,21 +677,13 @@ class AsyncIntel():
|
|
|
875
677
|
|
|
876
678
|
success_rate = (successful_assertions / total_assertions) * 100
|
|
877
679
|
|
|
680
|
+
status = "通过" if success_rate >= MIN_SUCCESS_RATE else "未通过"
|
|
878
681
|
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
'status':"通过",
|
|
682
|
+
self.eval_df.loc[len(self.eval_df)] = {"name":prompt_id,
|
|
683
|
+
'status':status,
|
|
882
684
|
"score":success_rate,
|
|
883
685
|
"total":str(total_assertions),
|
|
884
686
|
"bad_case":json.dumps(bad_case,ensure_ascii=False)}
|
|
885
|
-
return "通过", success_rate, str(total_assertions), json.dumps(bad_case,ensure_ascii=False),
|
|
886
|
-
else:
|
|
887
|
-
self.eval_df.loc[len(self.eval_df)] = {"name":prompt_id,
|
|
888
|
-
'status':"未通过",
|
|
889
|
-
"score":success_rate,
|
|
890
|
-
"total":str(total_assertions),
|
|
891
|
-
"bad_case":json.dumps(bad_case,ensure_ascii=False)}
|
|
892
|
-
return "未通过",success_rate, str(total_assertions), json.dumps(bad_case,ensure_ascii=False),
|
|
893
687
|
|
|
894
688
|
|
|
895
689
|
async def function_eval(self,
|
|
@@ -908,12 +702,8 @@ class AsyncIntel():
|
|
|
908
702
|
# TODO llm 评价 eval
|
|
909
703
|
"""
|
|
910
704
|
async with create_async_session(self.engine) as session:
|
|
911
|
-
|
|
705
|
+
await self.get_prompt_safe(prompt_id=prompt_id,
|
|
912
706
|
session=session)
|
|
913
|
-
if prompt_result is None:
|
|
914
|
-
raise IntellectRemoveError("不存在的prompt_id")
|
|
915
|
-
if prompt_result.action_type != "inference":
|
|
916
|
-
raise IntellectRemoveError("请在inference模式下使用次类")
|
|
917
707
|
|
|
918
708
|
if database_url:
|
|
919
709
|
eval_engine = create_async_engine(database_url, echo=False,
|
|
@@ -941,7 +731,6 @@ class AsyncIntel():
|
|
|
941
731
|
OutputFormat = OutputFormat,
|
|
942
732
|
ExtraFormats = ExtraFormats,
|
|
943
733
|
version = version,
|
|
944
|
-
inference_save_case = False,
|
|
945
734
|
ConTent_Function = ConTent_Function,
|
|
946
735
|
AConTent_Function = AConTent_Function,
|
|
947
736
|
)
|