pro-craft 0.1.23__tar.gz → 0.1.25__tar.gz

{pro_craft-0.1.23 → pro_craft-0.1.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pro-craft
-Version: 0.1.23
+Version: 0.1.25
 Summary: Add your description here
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown

{pro_craft-0.1.23 → pro_craft-0.1.25}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pro-craft"
-version = "0.1.23"
+version = "0.1.25"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"

{pro_craft-0.1.23 → pro_craft-0.1.25}/src/pro_craft/file_manager.py

@@ -7,7 +7,7 @@ from pro_craft import Intel
 from enum import Enum
 import json
 
-from pro_craft.database import Base, Prompt
+from pro_craft.database import Prompt
 from sqlalchemy import create_engine, Column, Integer, String
 from sqlalchemy.orm import sessionmaker, declarative_base
 from pro_craft.utils import create_session
@@ -30,7 +30,7 @@ class ContentManager():
         self.qdrant = QdrantManager(host = "localhost")
         self.neo = None
 
-    
+
     @intel.intellect_2(IntellectType.inference,
                     prompt_id = "db_help_001",
                     demand="""

{pro_craft-0.1.23 → pro_craft-0.1.25}/src/pro_craft/prompt_craft/async_.py

@@ -16,9 +16,16 @@ from datetime import datetime
 from pro_craft.utils import extract_
 import asyncio
 import re
-
+from pydantic import BaseModel, ValidationError, field_validator
 from sqlalchemy import select, desc
 from json.decoder import JSONDecodeError
+from pro_craft.database import SyncMetadata
+from datetime import datetime, timedelta
+from datetime import datetime, timedelta
+from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
+from sqlalchemy import select, and_ # 引入 select 和 and_
+from sqlalchemy.orm import class_mapper # 用于检查对象是否是持久化的
+
 
 class IntellectRemoveFormatError(Exception):
     pass
@@ -26,6 +33,7 @@ class IntellectRemoveFormatError(Exception):
 class IntellectRemoveError(Exception):
     pass
 
+BATCH_SIZE = 100
 
 def slog(s, target: str = "target",logger = None):
     COLOR_GREEN = "\033[92m"
@@ -62,6 +70,64 @@ def fix_broken_json_string(broken_json_str):
     return fixed_json_str
 
 
+# def get_last_sync_time(target_session) -> datetime:
+#     """从目标数据库获取上次同步时间"""
+#     metadata_entry = target_session.query(SyncMetadata).filter_by(table_name="sync_metadata").first()
+#     if metadata_entry:
+#         return metadata_entry.last_sync_time
+#     return datetime(1970, 1, 1) # 默认一个很早的时间
+
+# def update_last_sync_time(target_session, new_sync_time: datetime):
+#     """更新目标数据库的上次同步时间"""
+#     metadata_entry = target_session.query(SyncMetadata).filter_by(table_name="sync_metadata").first()
+#     if metadata_entry:
+#         metadata_entry.last_sync_time = new_sync_time
+#     else:
+#         # 如果不存在，则创建
+#         new_metadata = SyncMetadata(table_name="sync_metadata", last_sync_time=new_sync_time)
+#         target_session.add(new_metadata)
+#     target_session.commit()
+#     print(f"Updated last sync time to: {new_sync_time}")
+
+
+async def get_last_sync_time(target_session: AsyncSession) -> datetime:
+    """从目标数据库获取上次同步时间"""
+    # 修正点：使用 select() 和 execute()
+    result = await target_session.execute(
+        select(SyncMetadata).filter_by(table_name="sync_metadata")
+    )
+    metadata_entry = result.scalar_one_or_none() # 获取单个对象或 None
+
+    if metadata_entry:
+        return metadata_entry.last_sync_time
+    return datetime(1970, 1, 1) # 默认一个很早的时间
+
+
+# from your_module import SyncMetadata # 假设 SyncMetadata 已导入
+# from sqlalchemy import select # 确保引入 select
+
+async def update_last_sync_time(target_session: AsyncSession, new_sync_time: datetime):
+    """更新目标数据库的上次同步时间"""
+    # 修正点：使用 select() 和 execute()
+    result = await target_session.execute(
+        select(SyncMetadata).filter_by(table_name="sync_metadata")
+    )
+    metadata_entry = result.scalar_one_or_none()
+
+    if metadata_entry:
+        metadata_entry.last_sync_time = new_sync_time
+    else:
+        # 如果不存在，则创建
+        new_metadata = SyncMetadata(table_name="sync_metadata", last_sync_time=new_sync_time)
+        target_session.add(new_metadata)
+    
+    # 异步提交事务
+    await target_session.commit() # TODO
+    print(f"Updated last sync time to: {new_sync_time}")
+
+
+
+
 class IntellectType(Enum):
     train = "train"
     inference = "inference"
@@ -146,6 +212,101 @@ class AsyncIntel():
 
         return specific_prompt
 
+    async def sync_prompt_data_to_database(self,database_url:str):
+        target_engine = create_async_engine(database_url, echo=False)
+        async with target_engine.begin() as conn:
+            await conn.run_sync(PromptBase.metadata.create_all)
+            
+        async with create_async_session(self.engine) as source_session:
+            async with create_async_session(target_engine) as target_session:
+            
+                last_sync_time = await get_last_sync_time(target_session)
+                print(f"Starting sync for sync_metadata from: {last_sync_time}")
+
+
+                processed_count = 0
+                #2 next_sync_watermark = last_sync_time
+                current_batch_max_updated_at = last_sync_time
+
+                while True:
+                    source_results = await source_session.execute(
+                        select(Prompt)
+                        .filter(Prompt.timestamp > last_sync_time)
+                        .order_by(Prompt.timestamp.asc(), Prompt.id.asc())
+                        .limit(BATCH_SIZE)
+                    )
+                    records_to_sync = source_results.scalars().all()
+                    if not records_to_sync:
+                        break # 没有更多记录了
+
+                    #2 max_timestamp_in_batch = datetime(1970, 1, 1) # 初始化为最早时间
+
+                    # 准备要插入或更新到目标数据库的数据
+                    for record in records_to_sync:
+                        # 查找目标数据库中是否存在该ID的记录
+                        # 这里的 `User` 模型会对应到 target_db.users
+                        target_prompt_result = await target_session.execute(
+                            select(Prompt).filter_by(id=record.id) # 假设 prompt_id 是唯一标识符
+                        )
+                        target_prompt = target_prompt_result.scalar_one_or_none()
+                        
+                        if target_prompt:
+                            # 如果存在，则更新
+                            target_prompt.prompt_id = record.prompt_id
+                            target_prompt.version = record.version
+                            target_prompt.timestamp = record.timestamp
+                            target_prompt.prompt = record.prompt
+                            target_prompt.use_case = record.use_case
+                            target_prompt.action_type = record.action_type
+                            target_prompt.demand = record.demand
+                            target_prompt.score = record.score
+                            target_prompt.is_deleted = record.is_deleted
+                        else:
+                            # 如果不存在，则添加新记录
+                            # 注意：这里需要创建一个新的User实例，而不是直接添加源数据库的record对象
+                            new_prompt = Prompt(
+                                prompt_id=record.prompt_id, 
+                                version=record.version,
+                                timestamp=record.timestamp,
+                                prompt = record.prompt,
+                                use_case = record.use_case,
+                                action_type = record.action_type,
+                                demand = record.demand,
+                                score = record.score,
+                                is_deleted = record.is_deleted
+                                )
+                            target_session.add(new_prompt)
+                        
+                        # 记录当前批次最大的 updated_at
+                        #2 
+                        # if record.timestamp > max_timestamp_in_batch:
+                        #     max_timestamp_in_batch = record.timestamp
+                        if record.timestamp > current_batch_max_updated_at:
+                            current_batch_max_updated_at = record.timestamp
+
+
+                    await target_session.commit() 
+                    processed_count += len(records_to_sync)
+                    print(f"Processed {len(records_to_sync)} records. Total processed: {processed_count}")
+
+                    #2 next_sync_watermark = max_timestamp_in_batch + timedelta(microseconds=1)
+                    last_sync_time = current_batch_max_updated_at + timedelta(microseconds=1) 
+
+                    
+                    if len(records_to_sync) < BATCH_SIZE: # 如果查询到的记录数小于批次大小，说明已经处理完所有符合条件的记录
+                        break
+
+                if processed_count > 0:
+                    # 最终更新last_sync_time到数据库，确保记录的是所有已处理记录中最新的一个
+                    await update_last_sync_time(target_session, current_batch_max_updated_at + timedelta(microseconds=1))
+
+                    #2 await update_last_sync_time(target_session, next_sync_watermark)
+
+                    await target_session.commit() # 确保最终的 metadata 更新也被提交
+                else:
+                    print("No new records to sync.")
+
+
     async def get_prompts_from_sql(self,
                              prompt_id: str,
                              version = None,
@@ -244,7 +405,7 @@ class AsyncIntel():
         chat_history = s_prompt or system_result
         await self.save_prompt_increment_version(prompt_id,
                                 new_prompt = chat_history,
-                                use_case = " summary ",
+                                use_case = "",
                                 score = 60,
                                 session = session)
         
@@ -295,7 +456,7 @@ class AsyncIntel():
             new_prompt = prompt_
         await self.save_prompt_increment_version(prompt_id = prompt_id,
                             new_prompt = new_prompt,
-                            use_case = " finetune ",
+                            use_case = "",
                             score = 60,
                             session = session)
 
@@ -335,10 +496,6 @@ class AsyncIntel():
                 return "init"
 
 
-            
-            
-
-
 
     async def intellect_remove(self,
                     input_data: dict | str,
@@ -346,6 +503,7 @@ class AsyncIntel():
                     prompt_id: str,
                     version: str = None,
                     inference_save_case = True,
+                    change_case = False,
                     ):
         if isinstance(input_data,dict):
             input_ = json.dumps(input_data,ensure_ascii=False)
@@ -361,7 +519,7 @@ class AsyncIntel():
             prompt = result_obj.prompt
             if result_obj.action_type == "inference":
                 # 直接推理即可
-                ai_result = await self.llm.aproduct(prompt + output_format + "\n-----input----\n" +  input_)
+                ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" +  input_)
                 if inference_save_case:
                     await self.save_use_case_by_sql(prompt_id,
                                         use_case = input_,
@@ -397,10 +555,35 @@ class AsyncIntel():
                 #                         score = 60,
                 #                         session = session)
 
-                if input_ == before_input:
-                    new_prompt = prompt + "\nuser:" + demand
-                else:
-                    new_prompt = prompt + "\nuser:" + input_
+
+                # version 2
+
+                # if input_ == before_input:
+                #     new_prompt = prompt + "\nuser:" + demand
+                # else:
+                #     new_prompt = prompt + "\nuser:" + input_
+
+                # ai_result = await self.llm.aproduct(new_prompt + output_format)
+
+                # save_new_prompt = new_prompt + "\nassistant:\n" + ai_result
+
+
+                # await self.save_prompt_increment_version(
+                #     prompt_id, 
+                #     new_prompt=save_new_prompt,
+                #     use_case = input_,
+                #     action_type = "inference",
+                #     score = 60,
+                #     session = session)
+                
+                if before_input == "" or change_case is True:
+                    result_obj.use_case = input_
+                    await session.commit()
+                    # 查询上一条, 将before_input 更新位input_
+                    prompt += input_
+
+                # 使用更新后的数据进行后续步骤
+                new_prompt = prompt + "\nuser:" + demand
 
                 ai_result = await self.llm.aproduct(new_prompt + output_format)
 
@@ -411,6 +594,7 @@ class AsyncIntel():
                     prompt_id, 
                     new_prompt=save_new_prompt,
                     use_case = input_,
+                    action_type = "inference",
                     score = 60,
                     session = session)
     
@@ -420,7 +604,7 @@ class AsyncIntel():
                             prompt = prompt,
                             session = session
                             )
-                ai_result = await self.llm.aproduct(prompt + output_format + "\n-----input----\n" +  input_)
+                ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" +  input_)
 
             elif result_obj.action_type == "finetune":
                 demand = result_obj.demand
@@ -430,12 +614,12 @@ class AsyncIntel():
                                             demand = demand,
                                             session = session
                                             )
-                ai_result = await self.llm.aproduct(prompt + output_format + "\n-----input----\n" +  input_)
+                ai_result = await self.llm.aproduct(prompt + output_format + "\nuser:" +  input_)
             elif result_obj.action_type == "patch":
                 demand = result_obj.demand
                 assert demand
                 chat_history = prompt + demand
-                ai_result = await self.llm.aproduct(chat_history + output_format + "\n-----input----\n" +  input_)
+                ai_result = await self.llm.aproduct(chat_history + output_format + "\nuser:" +  input_)
                 self.save_prompt_increment_version(prompt_id, 
                                                    chat_history,
                                                     use_case = input_,
@@ -621,6 +805,13 @@ class AsyncIntel():
 
             except JSONDecodeError as e:
                 raise IntellectRemoveFormatError(f"prompt_id: {prompt_id} 在生成后做json解析时报错") from e
+        
+        except ValidationError as e:
+            err_info = e.errors()[0]
+            raise IntellectRemoveFormatError(f"{err_info["type"]}: 属性:{err_info['loc']}, 发生了如下错误: {err_info['msg']}, 格式校验失败, 当前输入为: {err_info['input']} 请检查") from e
+
+        except Exception as e:
+            raise Exception(f"Error {prompt_id} : {e}") from e
 
         return ai_result
     
@@ -636,6 +827,8 @@ class AsyncIntel():
         async with create_async_session(self.engine) as session:
             prompt_result = await self.get_prompts_from_sql(prompt_id=prompt_id,
                                                                    session=session)
+            if prompt_result is None:
+                raise IntellectRemoveError("不存在的prompt_id")
         if prompt_result.action_type != "inference":
             input_datas = input_datas[:1]
         tasks = []

{pro_craft-0.1.23 → pro_craft-0.1.25}/src/pro_craft/prompt_craft/sync.py

@@ -22,6 +22,8 @@ BATCH_SIZE = 1000
 class IntellectRemoveFormatError(Exception):
     pass
 
+class IntellectRemoveError(Exception):
+    pass
 
 def slog(s, target: str = "target",logger = None):
     COLOR_GREEN = "\033[92m"
@@ -354,17 +356,28 @@ class Intel():
         # 查看是否已经存在
         with create_session(self.engine) as session:
             latest_prompt = self.get_prompts_from_sql(prompt_id=prompt_id,session=session)
-
-            self.save_prompt_increment_version(prompt_id=latest_prompt.prompt_id,
-                                    new_prompt = latest_prompt.prompt,
-                                    use_case = latest_prompt.use_case,
-                                    action_type=action_type,
+               
+            if latest_prompt:
+                self.save_prompt_increment_version(prompt_id=latest_prompt.prompt_id,
+                                        new_prompt = latest_prompt.prompt,
+                                        use_case = latest_prompt.use_case,
+                                        action_type=action_type,
+                                        demand=demand,
+                                        score=latest_prompt.score,
+                                        session=session
+                                        )
+                
+                return "success"
+            else:
+                self.save_prompt_increment_version(prompt_id=prompt_id,
+                                    new_prompt = demand,
+                                    use_case = "init",
+                                    action_type="inference",
                                     demand=demand,
-                                    score=latest_prompt.score,
+                                    score=60,
                                     session=session
                                     )
-            
-            return "success"
+                return "init"
 
     def intellect_remove(self,
                     input_data: dict | str,
@@ -393,15 +406,7 @@ class Intel():
             result_obj = self.get_prompts_from_sql(prompt_id=prompt_id,session=session)
 
             if result_obj is None:
-                self.save_prompt_increment_version(
-                    prompt_id = prompt_id,
-                    new_prompt = "做一些处理",
-                    use_case = input_,
-                    score = 60,
-                    session = session
-                )
-                ai_result = "初始化完成"
-                return ai_result
+                raise IntellectRemoveError("不存在的prompt_id")
             prompt = result_obj.prompt
             if result_obj.action_type == "inference":
                 # 直接推理即可

{pro_craft-0.1.23 → pro_craft-0.1.25}/src/pro_craft/server/router/prompt.py

@@ -1,13 +1,15 @@
 
 
 from fastapi import APIRouter
-from pro_craft import Intel
+from pro_craft import Intel,AsyncIntel
 from pro_craft.utils import create_session
 
 def create_router(database_url: str,
                   slave_database_url: str,
-                  model_name: str):
+                  model_name: str,
+                  logger = None):
     """
+    # TODO 整理改为异步
     创建一个包含 ProCraft 路由的 FastAPI APIRouter 实例。
 
     Args:
@@ -20,9 +22,10 @@ def create_router(database_url: str,
         APIRouter: 配置好的 FastAPI APIRouter 实例。
     """
 
-    intels = Intel(
+    intels = AsyncIntel(
         database_url=database_url,
-        model_name=model_name
+        model_name=model_name,
+        logger=logger
         )
 
     router = APIRouter(
@@ -32,7 +35,7 @@ def create_router(database_url: str,
     @router.get("/push_order",
                 description="可选 train,inference,summary,finetune,patch",)
     async def push_order(demand: str, prompt_id: str, action_type: str = "train"):
-        result = intels.push_action_order(
+        result = await intels.push_action_order(
             demand=demand,
             prompt_id=prompt_id,
             action_type=action_type
@@ -42,7 +45,7 @@ def create_router(database_url: str,
     @router.get("/get_latest_prompt")
     async def get_latest_prompt(prompt_id: str):
         with create_session(intels.engine) as session:
-            result = intels.get_prompts_from_sql(
+            result = await intels.get_prompts_from_sql(
                 prompt_id=prompt_id,
                 session=session
             )
@@ -50,20 +53,20 @@ def create_router(database_url: str,
 
     @router.get("/sync_database")
     async def sync_database():
-        result = intels.sync_prompt_data_to_database(slave_database_url)
+        result = await intels.sync_prompt_data_to_database(slave_database_url)
         return {"message": "success","result":result}
     
 
     @router.get("/roll_back")
     async def roll_back(prompt_id:str,version:str):
         with create_session(intels.engine) as session:
-            result = intels.get_prompts_from_sql(
+            result = await intels.get_prompts_from_sql(
                 prompt_id=prompt_id,
                 version = version,
                 session=session
             )
             assert result.version == version
-            intels.save_prompt_increment_version(
+            await intels.save_prompt_increment_version(
                             prompt_id = prompt_id,
                             new_prompt = result.prompt,
                             use_case = result.use_case,

{pro_craft-0.1.23 → pro_craft-0.1.25}/src/pro_craft.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pro-craft
-Version: 0.1.23
+Version: 0.1.25
 Summary: Add your description here
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown