pro-craft 0.1.54__py3-none-any.whl → 0.1.55__py3-none-any.whl

pro_craft/prompt_craft/async_.py

@@ -25,6 +25,7 @@ from datetime import datetime, timedelta
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
 from sqlalchemy import select, and_ # 引入 select 和 and_
 from sqlalchemy.orm import class_mapper # 用于检查对象是否是持久化的
+from datetime import datetime, timedelta
 
 from tqdm.asyncio import tqdm
 import pandas as pd
@@ -333,6 +334,8 @@ class AsyncIntel():
 
     async def get_use_case(self,
                              target_prompt_id: str,
+                             start_time: datetime = None,  # 新增：开始时间
+                             end_time: datetime = None,    # 新增：结束时间
                              session = None
                             ):
         """
@@ -341,6 +344,11 @@ class AsyncIntel():
         stmt = select(UseCase).filter(UseCase.is_deleted == 0,
                                       UseCase.prompt_id == target_prompt_id)
         
+        if start_time:
+            stmt = stmt.filter(UseCase.created_at >= start_time)  # 假设你的UseCase模型有一个created_at字段
+
+        if end_time:
+            stmt = stmt.filter(UseCase.created_at <= end_time)
         result = await session.execute(stmt)
         # use_case = result.scalars().one_or_none()
         use_case = result.scalars().all()
@@ -665,8 +673,7 @@ class AsyncIntel():
             output_format = ""
 
         if logger:
-            logger.info(f'{type(input_data)} $ intellect输入 input_data $ {input_data}')
-            logger.info(f'{type(output_format)} $ intellect输入 output_format $ {output_format}')
+            logger.info(f'{type(input_data)} $ intellect-输入 $ {input_data} \noutput_format: \n{output_format}')
 
         ai_result = await self.intellect(
                     input_data=input_data,
@@ -792,6 +799,122 @@ class AsyncIntel():
             if prompt_result.action_type != "inference":
                 raise IntellectRemoveError("请在inference模式下使用次类")
 
+        if database_url:
+            eval_engine = create_async_engine(database_url, echo=False,
+                                pool_size=10,        # 连接池中保持的连接数
+                                max_overflow=20,     # 当pool_size不够时，允许临时创建的额外连接数
+                                pool_recycle=3600,   # 每小时回收一次连接
+                                pool_pre_ping=True,  # 使用前检查连接活性
+                                pool_timeout=30      # 等待连接池中连接的最长时间（秒）
+                                        )
+        else:
+            eval_engine = self.engine
+        async with create_async_session(eval_engine) as eval_session:
+            # start = datetime(2023, 1, 1, 10, 0, 0)
+            # end = datetime(2023, 1, 15, 12, 30, 0)
+            use_cases = await self.get_use_case(target_prompt_id=prompt_id,session=eval_session,
+                                                start_time=None,
+                                                end_time=None,)
+
+            total_assertions = len(use_cases)
+            result_cases = []
+
+            async def evals_func(use_case,prompt_id,OutputFormat,ExtraFormats,version):
+                try:
+
+                    # 这里将参数传入
+                    ai_result = await self.intellect_format(
+                        input_data = use_case.use_case,
+                        prompt_id = prompt_id,
+                        OutputFormat = OutputFormat,
+                        ExtraFormats = ExtraFormats,
+                        version = version,
+                        inference_save_case = False,
+                        ConTent_Function = ConTent_Function,
+                        AConTent_Function = AConTent_Function,
+                    )
+
+                    result_cases.append({"type":"Successful","case":use_case.use_case,"reply":f"pass"})
+                    use_case.output = json.dumps(ai_result,ensure_ascii=False,indent=4)
+
+
+                except IntellectRemoveFormatError as e:
+                    result_cases.append({"type":"FAILED","case":use_case.use_case,"reply":f"{e}"})
+                    use_case.output = f"{"FAILED"}-{e}"
+                    use_case.faired_time +=1
+
+                except Exception as e: # 捕获其他可能的错误
+                    result_cases.append({"type":"FAILED","case":use_case.use_case,"reply":f"Exp {e}"})
+                    use_case.output = f"{"FAILED"}-{e}"
+                    use_case.faired_time +=1
+
+
+            tasks = []
+            for use_case in use_cases:
+                tasks.append(
+                    evals_func(
+                        use_case = use_case,
+                        prompt_id = prompt_id,
+                        OutputFormat = OutputFormat,
+                        ExtraFormats = ExtraFormats,
+                        version = version
+                    )
+                )
+            await tqdm.gather(*tasks,total=len(tasks))
+            # await asyncio.gather(*tasks, return_exceptions=False)
+
+            await eval_session.commit()
+
+            successful_assertions = 0
+            bad_case = []
+            for i in result_cases:
+                if i['type'] == "Successful":
+                    successful_assertions += 1
+                else:
+                    bad_case.append(i)
+
+            success_rate = (successful_assertions / total_assertions) * 100
+
+
+            if success_rate >= MIN_SUCCESS_RATE:
+                self.eval_df.loc[len(self.eval_df)] = {"name":prompt_id,
+                                                   'status':"通过",
+                                                   "score":success_rate,
+                                                   "total":str(total_assertions),
+                                                   "bad_case":json.dumps(bad_case,ensure_ascii=False)}
+                return "通过", success_rate, str(total_assertions), json.dumps(bad_case,ensure_ascii=False),
+            else:
+                self.eval_df.loc[len(self.eval_df)] = {"name":prompt_id,
+                                                   'status':"未通过",
+                                                   "score":success_rate,
+                                                   "total":str(total_assertions),
+                                                   "bad_case":json.dumps(bad_case,ensure_ascii=False)}
+                return "未通过",success_rate, str(total_assertions), json.dumps(bad_case,ensure_ascii=False),
+
+
+    async def function_eval(self,
+                    OutputFormat: object,
+                    prompt_id: str,
+                    database_url = None,
+                    ExtraFormats: list[object] = [],
+                    version: str = None,
+                    MIN_SUCCESS_RATE = 80.0,
+                    ConTent_Function = None,
+                    AConTent_Function = None,
+                    ):
+        """
+        ConTent_Function: 
+        # TODO 人类评价 eval
+        # TODO llm 评价 eval
+        """
+        async with create_async_session(self.engine) as session:
+            prompt_result = await self.get_prompt_safe(prompt_id=prompt_id,
+                                                                   session=session)
+            if prompt_result is None:
+                raise IntellectRemoveError("不存在的prompt_id")
+            if prompt_result.action_type != "inference":
+                raise IntellectRemoveError("请在inference模式下使用次类")
+
         if database_url:
             eval_engine = create_async_engine(database_url, echo=False,
                                 pool_size=10,        # 连接池中保持的连接数
@@ -880,6 +1003,7 @@ class AsyncIntel():
                                                    "bad_case":json.dumps(bad_case,ensure_ascii=False)}
                 return "未通过",success_rate, str(total_assertions), json.dumps(bad_case,ensure_ascii=False),
 
+
     def draw_data(self,save_html_path = ""):
         df = self.eval_df
         # --- 可视化部分 ---

{pro_craft-0.1.54.dist-info → pro_craft-0.1.55.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pro-craft
-Version: 0.1.54
+Version: 0.1.55
 Summary: Add your description here
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{pro_craft-0.1.54.dist-info → pro_craft-0.1.55.dist-info}/RECORD

@@ -6,7 +6,7 @@ pro_craft/utils.py,sha256=R1DFkS4dsm5dIhg8lLTgBBvItvIYyyojROdh-ykqiYk,5250
 pro_craft/code_helper/coder.py,sha256=L6pRQr0pYRIHrMFZ4-pO_tZf1koxgGgF3L7Vl-GIyjM,24687
 pro_craft/code_helper/designer.py,sha256=3gyCqrjcw61sHzDjUPKhL1LOAE8xWLLbNT8NlK2mFLc,4739
 pro_craft/prompt_craft/__init__.py,sha256=83ruWO1Oci-DWvdVhPqcQrgdZTNfbmK72VQCkWASk7A,80
-pro_craft/prompt_craft/async_.py,sha256=AkHU06zJE0gVLJC1s_5ViQ3htO0hzaqLFkzpIjieh4w,43191
+pro_craft/prompt_craft/async_.py,sha256=KPWhzlFmLVm93-oRPvMJQlzzPX00qbjlvIUVaEfjiNI,49079
 pro_craft/prompt_craft/new.py,sha256=ULjGGl95vmHrOs7XECJGlaqj1NE9BypE5WnFYhGugRY,25903
 pro_craft/prompt_craft/sync.py,sha256=4bms8Qvzq5QqgwHWwiyjrcl7hdkSqE7Kne5s3Ex8bBU,26217
 pro_craft/server/mcp/__init__.py,sha256=4dbl-lFcm0r2tkOP04OxqiZG2jR-rqF181qi2AfU6UA,123
@@ -15,7 +15,7 @@ pro_craft/server/mcp/prompt.py,sha256=zZ58zb422vhwoL_uYCVBdOwOR-xy69MLQ5tyVfbpBl
 pro_craft/server/router/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pro_craft/server/router/models.py,sha256=o60_qHjFIHwH3Pp6jNmg3V23GyFeXJXH-V6uBVoztXQ,1340
 pro_craft/server/router/prompt.py,sha256=goKipoCX62BqtNXE2DSzpmqM2yzs2wvSzhvoDuuG5Ik,6862
-pro_craft-0.1.54.dist-info/METADATA,sha256=Mz50Pcur2gMzh1--beNQsEYslXnaqfswZvjz6TX32jc,1782
-pro_craft-0.1.54.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pro_craft-0.1.54.dist-info/top_level.txt,sha256=yqYDHArnYMWpeCxkmGRwlL6sJtxiOUnYylLDx9EOgFg,10
-pro_craft-0.1.54.dist-info/RECORD,,
+pro_craft-0.1.55.dist-info/METADATA,sha256=XXKESMgyd07rmPPBGF5BirCvTOAsPJUpgmo87b6T7rI,1782
+pro_craft-0.1.55.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pro_craft-0.1.55.dist-info/top_level.txt,sha256=yqYDHArnYMWpeCxkmGRwlL6sJtxiOUnYylLDx9EOgFg,10
+pro_craft-0.1.55.dist-info/RECORD,,