ws-bom-robot-app 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -11,6 +11,8 @@ class Settings(BaseSettings):
     robot_data_db_folder_src: str = 'src'
     robot_data_db_folder_out: str = 'out'
     robot_data_db_folder_store: str = 'store'
+    robot_data_db_retention_days: float = 60
+    robot_task_retention_days: float = 1
     robot_cms_host: str = ''
     robot_cms_auth: str = ''
     robot_cms_db_folder: str = 'llmVectorDb'

ws_bom_robot_app/cron_manager.py

@@ -0,0 +1,93 @@
+import os
+import threading
+import schedule, time
+import asyncio
+from fastapi import APIRouter
+from ws_bom_robot_app.task_manager import task_manager
+from ws_bom_robot_app.llm.utils.kb import kb_cleanup_data_file
+from ws_bom_robot_app.util import _log
+
+class RecurringJob():
+    def __init__(self, interval: int, job_func, tags: list[str]):
+        self.interval = interval
+        self.job_func = job_func
+        self.is_coroutine = asyncio.iscoroutinefunction(job_func)
+        self.job_func = job_func
+        self.tags = tags
+    def run(self):
+        if self.is_coroutine:
+            schedule.every(self.interval).seconds.do(self._run_async_job).tag(*self.tags)
+        else:
+            schedule.every(self.interval).seconds.do(self.job_func).tag(*self.tags)
+    async def _run_async_job(self):
+        await self.job_func()
+
+class CronManager:
+
+    _list: dict[str, RecurringJob] = {
+        'cleanup-task': RecurringJob(5*60, task_manager.cleanup_task, tags=["cleanup","cleanup-task"]),
+        'cleanup-data': RecurringJob(180*60, kb_cleanup_data_file, tags=["cleanup","cleanup-data"]),
+    }
+
+    def __init__(self):
+        self.jobs: dict[str, RecurringJob] = CronManager._list
+        self.__scheduler_is_running = False
+    def add_job(self, name:str, job: RecurringJob):
+        job = {name: job}
+        self.jobs.append(job)
+        return job
+    def run_pending(self):
+        return schedule.run_pending()
+    def run_all(self):
+        return schedule.run_all()
+    def clear(self):
+        self.__scheduler_is_running = False
+        return schedule.clear()
+    def get_jobs(self):
+        return schedule.get_jobs()
+    def start(self):
+        def _target():
+            while self.__scheduler_is_running:
+                time.sleep(1)
+                self.run_pending()
+                time.sleep(59)
+            _log.info(f"__scheduler_is_running={self.__scheduler_is_running}")
+        #clear all jobs
+        self.clear()
+        #prepare jobs
+        for job in self.jobs.values():
+            job.run()
+        #start scheduler
+        if not self.__scheduler_is_running:
+            self.__scheduler_is_running = True
+            t = threading.Thread(target=_target,args=(),daemon=True)
+            t.start()
+
+cron_manager = CronManager()
+
+router = APIRouter(prefix="/api/cron", tags=["cron"])
+@router.get("/list")
+def _list():
+    def __format(job: schedule.Job) -> dict:
+        return {
+            "job": {'module':job.job_func.__module__,'name':job.job_func.__name__},
+            "at": job.at_time,
+            "interval": job.interval,
+            "last_run": job.last_run,
+            "next_run": job.next_run,
+            "tags": job.tags}
+    _list = cron_manager.get_jobs()
+    return [__format(_) for _ in _list]
+
+@router.get("/start")
+def _start():
+    cron_manager.start()
+@router.delete("/stop")
+def _stop():
+    return {"_": cron_manager.clear()}
+@router.get("/run/pending")
+def _run_pending():
+    return {"_": cron_manager.run_pending()}
+@router.get("/run/all")
+def _run_all():
+    return {"_": cron_manager.run_all()}

ws_bom_robot_app/llm/api.py

@@ -7,6 +7,7 @@ from ws_bom_robot_app.llm.main import invoke, stream, stream_none
 from ws_bom_robot_app.llm.vector_store.generator import kb, rules, kb_stream_file
 from ws_bom_robot_app.llm.tools.tool_manager import ToolManager
 from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
+from ws_bom_robot_app.task_manager import task_manager
 
 router = APIRouter(prefix="/api/llm", tags=["llm"])
 
@@ -38,6 +39,10 @@ async def _stream_raw_none(rq: StreamRequest) -> None:
 async def _kb(rq: KbRequest) -> VectorDbResponse:
     return await kb(rq)
 
+@router.post("/kb/task")
+async def _create_kb_task(rq: KbRequest):
+    return {"task_id": task_manager.create_task(kb(rq))}
+
 @router.post("/rules")
 def _rules(rq: RulesRequest) -> VectorDbResponse:
     return rules(rq)

ws_bom_robot_app/llm/models/kb.py

@@ -1,10 +1,12 @@
-import json, os , uuid, requests
+import aiofiles
+import json, os , uuid, httpx
 from enum import Enum
 from typing import Optional, Literal
 from pydantic import BaseModel, Field, AliasChoices, ConfigDict
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.json_loader import JsonLoader
 from ws_bom_robot_app.util import timer
+import asyncio
 
 class LlmKbIntegration(BaseModel):
   type: str = Field(..., validation_alias=AliasChoices("blockType","type"))
@@ -65,7 +67,7 @@ class LlmKbEndpoint(BaseModel):
     fields_mapping: LlmKbEndpointFieldsMapping = Field(validation_alias=AliasChoices("fieldsMapping","fields_mapping"))
 
 # Remapping Function
-def __remap_knowledgebase_file(filepath: str, mapping: LlmKbEndpointFieldsMapping) -> None:
+async def __remap_knowledgebase_file(filepath: str, mapping: LlmKbEndpointFieldsMapping) -> None:
     map_new_fields = mapping.new_fields or []
     map_replaced_fields = mapping.replaced_fields or []
     deleted_fields = mapping.deleted_fields or []
@@ -73,7 +75,7 @@ def __remap_knowledgebase_file(filepath: str, mapping: LlmKbEndpointFieldsMappin
     if all([not map_new_fields,not map_replaced_fields,not deleted_fields]):
         return
 
-    with open(filepath, 'r', encoding='utf8') as file:
+    async with aiofiles.open(filepath, 'r', encoding='utf8') as file:
         original_data = json.load(file)
 
     for item in original_data:
@@ -106,45 +108,102 @@ def __remap_knowledgebase_file(filepath: str, mapping: LlmKbEndpointFieldsMappin
         # New fields
         for field in map_new_fields:
             item[field.name] = field.value
+            async with aiofiles.open(filepath, 'w', encoding='utf8') as file:
+              await file.write(json.dumps(original_data, ensure_ascii=False, indent=4))
 
-    with open(filepath, 'w', encoding='utf8') as file:
+    async with aiofiles.open(filepath, 'w', encoding='utf8') as file:
         json.dump(original_data, file, ensure_ascii=False, indent=4)
 
 # Download External Endpoints
-@timer
-def load_endpoints(endpoints: list[LlmKbEndpoint], destination_directory: str) -> list[Document]:
-    _documents = []
-
-    for endpoint in endpoints:
-        headers = {}
-        if endpoint.authentication != ExternalEndpointAuthentication.NONE:
-          auth_formats = {
-              ExternalEndpointAuthentication.BASIC: lambda secret: f'Basic {secret}',
-              ExternalEndpointAuthentication.BEARER: lambda secret: f'Bearer {secret}',
-              ExternalEndpointAuthentication.CUSTOM: lambda secret: secret
-          }
-          headers['Authorization'] = auth_formats[endpoint.authentication](endpoint.auth_secret)
-        try:
-          response = requests.get(endpoint.endpoint_url, headers=headers)
-          response.raise_for_status()
-
-          mime_type = response.headers.get('content-type', None)
-          if mime_type.__contains__('application/json'):
-              filename = f"{uuid.uuid4()}.json"
-              file_path = os.path.join(destination_directory, filename)
-              with open(file_path, 'wb') as file:
-                  file.write(response.content)
-              __remap_knowledgebase_file(file_path, endpoint.fields_mapping)
-              _documents.extend(
-                  JsonLoader(
-                      file_path,
-                      meta_fields=[field.name for field in endpoint.fields_mapping.meta_fields] if endpoint.fields_mapping.meta_fields else []
-                      ).load())
-          else:
-              raise Exception(f"Unsupported content type {mime_type}")
-
-        except requests.exceptions.RequestException as e:
-            raise Exception(f"Failed to download file from endpoint [status {response.status_code}]: {endpoint.endpoint_url}") from e
-
-    return _documents
+#@timer
+async def load_endpoints(endpoints: list[LlmKbEndpoint], destination_directory: str) -> list[Document]:
+  _documents = []
+
+  async def process_endpoint(endpoint: LlmKbEndpoint):
+    headers = {}
+    if endpoint.authentication != ExternalEndpointAuthentication.NONE:
+      auth_formats = {
+        ExternalEndpointAuthentication.BASIC: lambda secret: f'Basic {secret}',
+        ExternalEndpointAuthentication.BEARER: lambda secret: f'Bearer {secret}',
+        ExternalEndpointAuthentication.CUSTOM: lambda secret: secret
+      }
+      headers['Authorization'] = auth_formats[endpoint.authentication](endpoint.auth_secret)
+    try:
+      async with httpx.AsyncClient() as client:
+        response = await client.get(endpoint.endpoint_url, headers=headers)
+        response.raise_for_status()
+
+        mime_type = response.headers.get('content-type', None)
+        if mime_type.__contains__('application/json'):
+          filename = f"{uuid.uuid4()}.json"
+          file_path = os.path.join(destination_directory, filename)
+          async with aiofiles.open(file_path, 'wb') as file:
+            await file.write(response.content)
+          await __remap_knowledgebase_file(file_path, endpoint.fields_mapping)
+          try:
+            documents = await JsonLoader(
+              file_path,
+              meta_fields=[field.name for field in endpoint.fields_mapping.meta_fields] if endpoint.fields_mapping.meta_fields else []
+            ).load()
+            _documents.extend(documents)
+            await aiofiles.os.remove(file_path)
+          except Exception as e:
+            raise Exception(f"Failed to load documents from endpoint: {endpoint.endpoint_url} [{e}]") from e
+        else:
+          raise Exception(f"Unsupported content type {mime_type}")
+    except httpx.HTTPStatusError as e:
+      raise Exception(f"Failed to download file from endpoint [status {e.response.status_code}]: {endpoint.endpoint_url}") from e
+    except httpx.RequestError as e:
+      raise Exception(f"Failed to download file from endpoint: {endpoint.endpoint_url} [{e.request.content}]") from e
+    except Exception as e:
+      raise Exception(f"Failed to download file from endpoint: {endpoint.endpoint_url} [{e}]") from e
+
+  await asyncio.gather(*[process_endpoint(endpoint) for endpoint in endpoints])
+
+  return _documents
+
+async def __remap_knowledgebase_file(filepath: str, mapping: LlmKbEndpointFieldsMapping) -> None:
+  map_new_fields = mapping.new_fields or []
+  map_replaced_fields = mapping.replaced_fields or []
+  deleted_fields = mapping.deleted_fields or []
+
+  if all([not map_new_fields, not map_replaced_fields, not deleted_fields]):
+    return
+
+  async with aiofiles.open(filepath, 'r', encoding='utf8') as file:
+    original_data = json.loads(await file.read())
+
+  for item in original_data:
+    # Replaced fields
+    for field in map_replaced_fields:
+      if '.' in field.src_name:
+        keys = field.src_name.split('.')
+        last_key = keys.pop()
+        obj = item
+        for key in keys:
+          obj = obj.get(key, None)
+        if obj is not None:
+          obj[field.dest_name] = obj.pop(last_key, None)
+      else:
+        item[field.dest_name] = item.pop(field.src_name, None)
+
+    # Deleted fields
+    for field in deleted_fields:
+      if '.' in field.name:
+        keys = field.name.split('.')
+        last_key = keys.pop()
+        obj = item
+        for key in keys:
+          obj = obj.get(key, None)
+        if obj is not None:
+          obj.pop(last_key, None)
+      else:
+        item.pop(field.name, None)
+
+    # New fields
+    for field in map_new_fields:
+      item[field.name] = field.value
+
+  async with aiofiles.open(filepath, 'w', encoding='utf8') as file:
+    await file.write(json.dumps(original_data, ensure_ascii=False, indent=4))
 

ws_bom_robot_app/llm/utils/faiss_helper.py

@@ -11,11 +11,13 @@ class FaissHelper():
     _embedding_model = "text-embedding-3-small"
     _CACHE: dict[str, FAISS] = {}
     @staticmethod
-    @timer
-    def create(documents: list[Document],folder_path:str,api_key:SecretStr) -> None:
-        FAISS.from_documents(documents, OpenAIEmbeddings(api_key=api_key, model=FaissHelper._embedding_model)).save_local(folder_path)
+    #@timer
+    async def create(documents: list[Document], folder_path: str, api_key: SecretStr) -> None:
+      embeddings = OpenAIEmbeddings(api_key=api_key, model=FaissHelper._embedding_model)
+      faiss_instance = await asyncio.to_thread(FAISS.from_documents, documents, embeddings)
+      await asyncio.to_thread(faiss_instance.save_local, folder_path)
     @staticmethod
-    @timer
+    #@timer
     def get_loader(folder_path:str,api_key:SecretStr) -> FAISS:
         """_summary_
 
@@ -36,7 +38,7 @@ class FaissHelper():
         return FaissHelper._CACHE[folder_path]
 
     @staticmethod
-    @timer
+    #@timer
     def get_retriever(folder_path:str,api_key:SecretStr,search_type=str, search_kwargs= dict[str,Any]) -> VectorStoreRetriever:
         """_summary_
 
@@ -50,7 +52,7 @@ class FaissHelper():
         _faiss = FaissHelper.get_loader(folder_path,api_key)
         return _faiss.as_retriever(search_type=search_type, search_kwargs=search_kwargs)
     @staticmethod
-    @atimer
+    #@atimer
     async def _combine_search(retrievers: list[VectorStoreRetriever], query:str) -> list[Document]:
         """_summary_
 
@@ -76,7 +78,7 @@ class FaissHelper():
         all_docs = _remove_duplicates([doc for docs in search_results for doc in docs])
         return all_docs
     @staticmethod
-    @atimer
+    #@atimer
     async def invoke(folder_path:str,api_key:SecretStr, query:str, search_type=str, search_kwargs= dict[str,Any]) -> list[Document]:
         """_summary_
         Args:

ws_bom_robot_app/llm/utils/kb.py

@@ -0,0 +1,34 @@
+import os
+from ws_bom_robot_app.config import config
+from datetime import datetime, timedelta
+from ws_bom_robot_app.util import _log
+
+def kb_cleanup_data_file() -> dict:
+    """
+    clean up old data files in the specified folder
+
+    Returns:
+    - Dictionary with cleanup statistics
+    """
+    _deleted_files = []
+    _freed_space = 0
+    folder = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out)
+
+    for root, dirs, files in os.walk(folder, topdown=False):
+        for file in files:
+            file_path = os.path.join(root, file)
+            file_stat = os.stat(file_path)
+            file_creation_time = datetime.fromtimestamp(file_stat.st_ctime)
+            if file_creation_time < datetime.now() - timedelta(days=config.robot_data_db_retention_days):
+                _freed_space += file_stat.st_size
+                os.remove(file_path)
+                _deleted_files.append(file_path)
+        if not os.listdir(root):
+            os.rmdir(root)
+
+    _log.info(f"Deleted {len(_deleted_files)} files; Freed space: {_freed_space / (1024 * 1024):.2f} MB")
+
+    return {
+        "deleted_files_count": len(_deleted_files),
+        "freed_space_mb": _freed_space / (1024 * 1024)
+    }

ws_bom_robot_app/llm/vector_store/generator.py

@@ -1,4 +1,5 @@
-import os, shutil
+import os, shutil, traceback
+import asyncio
 from fastapi import HTTPException
 from fastapi.responses import StreamingResponse
 from langchain_openai import OpenAIEmbeddings
@@ -8,16 +9,16 @@ from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
 from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
 from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
-from ws_bom_robot_app.util import timer,atimer
+from ws_bom_robot_app.util import timer
 
 @timer
-def rules(rq: RulesRequest) -> VectorDbResponse:
+async def rules(rq: RulesRequest) -> VectorDbResponse:
   api_key = rq.api_key()
   _config = rq.config()
   db_name = rq.out_name()
   store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
   try:
-    FaissHelper.create([Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path, api_key) #type: ignore
+    await FaissHelper.create([Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path, api_key) #type: ignore
     #FAISS.from_texts([json.dumps({"rule": rule}) for rule in rq.rules], OpenAIEmbeddings(api_key=api_key)).save_local(store_path) #type: ignore
     db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
     return VectorDbResponse(file = os.path.basename(db_file_path))
@@ -25,8 +26,10 @@ def rules(rq: RulesRequest) -> VectorDbResponse:
     os.removedirs(store_path)
     return VectorDbResponse(success = False, error = str(e))
 
-@atimer
-async def kb(rq: KbRequest):
+#@atimer
+async def kb(rq: KbRequest) -> VectorDbResponse:
+  import aiofiles
+  import aiofiles.os
   os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
   from ws_bom_robot_app.llm.models.kb import load_endpoints
   from ws_bom_robot_app.llm.utils.download import download_files
@@ -39,8 +42,8 @@ async def kb(rq: KbRequest):
   if all([not rq.files,not rq.endpoints,not rq.integrations]):
     return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
   else:
-    os.makedirs(src_path, exist_ok=True)
-    os.makedirs(working_path, exist_ok=True)
+    await aiofiles.os.makedirs(src_path, exist_ok=True)
+    await aiofiles.os.makedirs(working_path, exist_ok=True)
 
   loaders = Loader(working_path)
   filter_file_extensions = loaders.managed_file_extensions()
@@ -55,7 +58,9 @@ async def kb(rq: KbRequest):
           src_path, authorization=_config.robot_cms_auth)
       # copy files to working tmp folder
       for file in rq.files:
-        shutil.copy(os.path.join(src_path, os.path.basename(file)), os.path.join(working_path, os.path.basename(file)))
+        async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
+          async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
+            await dest_file.write(await src_file.read())
     except Exception as e:
       return VectorDbResponse(success = False, error = f"Failed to download file {e}")
 
@@ -63,34 +68,42 @@ async def kb(rq: KbRequest):
 
   if rq.endpoints:
     try:
-      documents.extend(load_endpoints(rq.endpoints, working_path))
+      documents.extend(await load_endpoints(rq.endpoints, working_path))
     except Exception as e:
-      return VectorDbResponse(success = False, error = f"{e}")
+      tb = traceback.format_exc()
+      return VectorDbResponse(success = False, error = f"Endpoint failure: {e} | {tb}")
 
   if rq.integrations:
+    tasks = []
     for integration in rq.integrations:
-      try:
-        documents.extend(IntegrationManager
-          .get_strategy(integration.type.lower(),working_path,integration.__pydantic_extra__) #type: ignore
-          .load()
-        )
-      except Exception as e:
-        return VectorDbResponse(success = False, error = f"Integration failure: {e}")
+      tasks.append(
+        IntegrationManager
+        .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
+        .load()
+      )
+    try:
+      integration_documents = await asyncio.gather(*tasks)
+      for docs in integration_documents:
+        documents.extend(docs)
+    except Exception as e:
+      tb = traceback.format_exc()
+      return VectorDbResponse(success=False, error=f"Integration failure: {e} | {tb}")
 
   try:
-    documents.extend(loaders.load())
+    documents.extend(await loaders.load())
   except Exception as e:
-    return VectorDbResponse(success = False, error = f"Failed to load files {e}")
+    tb = traceback.format_exc()
+    return VectorDbResponse(success = False, error = f"Failed to load files {e} | {tb}")
   finally:
-    shutil.rmtree(working_path)
+    await aiofiles.os.wrap(shutil.rmtree)(working_path)
 
   if documents and len(documents) > 0:
     try:
       store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
       # create db
-      FaissHelper.create(documents,store_path,api_key) #type: ignore
+      await FaissHelper.create(documents,store_path,api_key) #type: ignore
       # zip db
-      db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
+      db_file_path = await aiofiles.os.wrap(shutil.make_archive)(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
       return VectorDbResponse(file = os.path.basename(db_file_path))
     except Exception as e:
       return VectorDbResponse(success = False, error = str(e))
@@ -99,6 +112,7 @@ async def kb(rq: KbRequest):
 
 async def kb_stream_file(filename: str):
     from ws_bom_robot_app.config import config
+
     file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
     if not os.path.isfile(file_path):
         raise HTTPException(status_code=404, detail="File not found")

ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -14,6 +14,6 @@ class IntegrationStrategy(ABC):
   def working_subdirectory(self) -> str:
     pass
   @abstractmethod
-  @timer
+  #@timer
   def load(self) -> list[Document]:
     pass

ws_bom_robot_app/llm/vector_store/integration/sitemap.py

@@ -1,3 +1,5 @@
+import aiofiles
+import aiofiles.os
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
 from langchain_community.document_loaders.sitemap import SitemapLoader
 from langchain_community.document_transformers import MarkdownifyTransformer as markdownify
@@ -53,7 +55,7 @@ class Sitemap(IntegrationStrategy):
                 for _ in content.select(element):
                     _.decompose()
             return str(self._extract(content))
-    def load(self) -> list[Document]:
+    async def load(self) -> list[Document]:
         def _is_local(url: str) -> bool:
             return not url.startswith("http")
         def _remap_if_local(url: str) -> str:
@@ -66,8 +68,8 @@ class Sitemap(IntegrationStrategy):
                 parsing_function=self._parse,
                 is_local=_is_local(self.__sitemap_url)
             )
-            _docs = self._output(_loader.load())
+            _docs = self._output([document async for document in _loader.alazy_load()])
             if _is_local(self.__sitemap_url):
-              os.remove(_loader.web_path)
+              await aiofiles.os.remove(_loader.web_path)
             return _docs
         return []

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -1,4 +1,5 @@
 
+import asyncio
 from ws_bom_robot_app.config import config
 from typing import Any, Callable, Generator, Optional, Tuple
 from langchain_community.document_loaders import DirectoryLoader
@@ -72,7 +73,7 @@ class Loader():
   def managed_file_extensions() -> list[str]:
     return [k for k,v in Loader._list.items() if v is not None]
 
-  @timer
+  #@timer
   def __directory_loader(self) -> list[DirectoryLoader]:
     loader_configs = {}
     for ext, loader_config in Loader._list.items():
@@ -88,6 +89,7 @@ class Loader():
                 }
             loader_configs[loader_key]["glob_patterns"].append(f"**/*{ext}")
     loaders = []
+
     for loader_config in loader_configs.values():
         loaders.append(
           DirectoryLoader(
@@ -104,7 +106,7 @@ class Loader():
         )
     return loaders
 
-  @timer
-  def load(self) -> list[Document]:
+  #@timer
+  async def load(self) -> list[Document]:
     loaders = MergedDataLoader(self.__directory_loader())
-    return loaders.load()
+    return await asyncio.to_thread(loaders.load)

ws_bom_robot_app/llm/vector_store/loader/json_loader.py

@@ -2,6 +2,7 @@ import json
 from typing import Optional
 from langchain_core.documents import Document
 from langchain_community.document_loaders.base import BaseLoader
+import aiofiles
 
 class JsonLoader(BaseLoader):
   def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
@@ -9,16 +10,17 @@ class JsonLoader(BaseLoader):
     self.meta_fields = meta_fields
     self.encoding = encoding
 
-  def load(self) -> list[Document]:
-    with open(self.file_path, "r", encoding=self.encoding) as file:
-      data = json.load(file)
-    _list  = data if isinstance(data, list) else [data]
+  async def load(self) -> list[Document]:
+    async with aiofiles.open(self.file_path, "r", encoding=self.encoding) as file:
+      data = json.loads(await file.read())
+    _list = data if isinstance(data, list) else [data]
     return [
       Document(
         page_content=json.dumps(item),
         metadata={
           "source": self.file_path,
-          **{field: item.get(field) for field in self.meta_fields if item.get(field)}}
-        )
-        for item in _list
+          **{field: item.get(field) for field in self.meta_fields if item.get(field)}
+        }
+      )
+      for item in _list
     ]

ws_bom_robot_app/main.py

@@ -7,12 +7,18 @@ from fastapi.openapi.utils import get_openapi
 from ws_bom_robot_app.auth import authenticate
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.api import router as llm
+from ws_bom_robot_app.task_manager import router as task
+from ws_bom_robot_app.cron_manager import (
+    router as cron,
+    cron_manager)
 from ws_bom_robot_app.util import _log
 
 _uptime = datetime.datetime.now()
+cron_manager.start()
 app = FastAPI(redoc_url=None,docs_url=None,openapi_url=None)
-
 app.include_router(llm,dependencies=[Depends(authenticate)])
+app.include_router(task,dependencies=[Depends(authenticate)])
+app.include_router(cron,dependencies=[Depends(authenticate)])
 
 @app.get("/")
 async def root():

ws_bom_robot_app/task_manager.py

@@ -0,0 +1,120 @@
+import asyncio, os
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import TypeVar, Optional, Dict, Union
+from pydantic import BaseModel, ConfigDict, computed_field
+from uuid import uuid4
+from fastapi import APIRouter, HTTPException
+from ws_bom_robot_app.config import config
+
+T = TypeVar('T')
+
+class TaskMetaData(BaseModel):
+    start_time: datetime
+    end_time: Optional[datetime] = None
+    @computed_field
+    @property
+    def elapsed_time(self) -> Union[timedelta, None]:
+        return (datetime.now() if not self.end_time else self.end_time) - self.start_time
+    source: Optional[str] = None
+    pid: Optional[int] = None
+    model_config = ConfigDict(
+        json_encoders={
+            datetime: lambda v: v.isoformat(),
+            timedelta: lambda v: str(v)
+        }
+    )
+
+class TaskStatus(BaseModel):
+    class TaskStatusEnum(str, Enum):
+        pending = "pending"
+        completed = "completed"
+        failure = "failure"
+    task_id: str
+    status: TaskStatusEnum
+    #result: Optional[Dict[str, Any]] = None
+    result: Optional[T] = None
+    metadata: TaskMetaData = None
+    error: Optional[str] = None
+
+class TaskEntry(BaseModel):
+    id: str
+    task: Union[asyncio.Task, None] = None
+    status: TaskStatus = None
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.status = TaskStatus(
+            task_id=self.id,
+            status=TaskStatus.TaskStatusEnum.pending,
+            metadata=TaskMetaData(start_time=datetime.now(), source=str(self.task), pid=os.getpid())
+            )
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
+
+class TaskManager:
+    def __init__(self):
+        self.tasks: Dict[str, TaskEntry] = {}
+
+    def _task_done_callback(self, task_id: str):
+        def callback(task: asyncio.Task):
+            if _task := self.tasks.get(task_id):
+              try:
+                  result = _task.task.result()
+                  _task.status.status = TaskStatus.TaskStatusEnum.completed
+                  _task.status.result = result
+              except Exception as e:
+                  _task.status.status = TaskStatus.TaskStatusEnum.failure
+                  _task.status.error = str(e)
+              finally:
+                  _task.status.metadata.end_time = datetime.now()
+        return callback
+
+    def create_task(self, coroutine: asyncio.coroutines) -> str:
+        _task = asyncio.create_task(coroutine)
+        task = TaskEntry(id=str(uuid4()),task=_task)
+        task.task.add_done_callback(self._task_done_callback(task.id))
+        self.tasks[task.id] = task
+        return task.id
+
+    def get_task(self, task_id: str) -> TaskEntry | None:
+        if _task := self.tasks.get(task_id):
+            return _task
+        return None
+
+    def remove_task(self, task_id: str) -> None:
+        if task_id in self.tasks:
+            del self.tasks[task_id]
+
+    def cleanup_task(self):
+        for task_id in [task_id for task_id, task in self.tasks.items()
+                        if task.status.status in {TaskStatus.TaskStatusEnum.completed, TaskStatus.TaskStatusEnum.failure}
+                        and task.status.metadata.end_time < datetime.now() - timedelta(days=config.robot_task_retention_days)]:
+            self.remove_task(task_id)
+
+# global instance
+task_manager = TaskManager()
+
+router = APIRouter(prefix="/api/task", tags=["task"])
+@router.get("/status/{task_id}", response_model=TaskStatus)
+async def _status_task(task_id: str) -> TaskStatus:
+    task = task_manager.get_task(task_id)
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return task.status
+@router.get("/status")
+async def _status_task_list():
+    _status_task_list = []
+    for task in task_manager.tasks.values():
+        _task = task_manager.get_task(task.id)
+        _status_task_list.append(_task.status)
+    return _status_task_list
+@router.delete("/status/{task_id}")
+async def _remove_task(task_id: str):
+    task_manager.remove_task(task_id)
+    return {"success":"ok"}
+@router.delete("/cleanup")
+async def _remove_task_list():
+    task_manager.cleanup_task()
+    return {"success":"ok"}
+

{ws_bom_robot_app-0.0.5.dist-info → ws_bom_robot_app-0.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.5
+Version: 0.0.7
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -10,6 +10,8 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
+Requires-Dist: schedule
+Requires-Dist: aiofiles
 Requires-Dist: pydantic==2.9.2
 Requires-Dist: pydantic-settings==2.6.0
 Requires-Dist: fastapi[standard]==0.115.5

{ws_bom_robot_app-0.0.5.dist-info → ws_bom_robot_app-0.0.7.dist-info}/RECORD

@@ -1,19 +1,21 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
-ws_bom_robot_app/config.py,sha256=ITbrJzlBmr11hgzmyQMUAubNbX_H_xscOgY5u4rdtDA,1743
-ws_bom_robot_app/main.py,sha256=eqGA4wK-MVhnZMg97KLNRYP5-RfMEQwrvVLY1hkz5AU,3144
+ws_bom_robot_app/config.py,sha256=VUoE6MBpYnZ8FQvUjWOLCzfs35aUvpwrz4ROFVbO3AA,1831
+ws_bom_robot_app/cron_manager.py,sha256=p0l0ybwfNhPn-R0kr3sq4rmR9VW3XDrsOu1BgvapYKc,3163
+ws_bom_robot_app/main.py,sha256=OZ0s2L2fKEwH46Vnt5KGEbuq-udEJH4WsinR2Ws8B4E,3433
+ws_bom_robot_app/task_manager.py,sha256=O6ZiQto0SxqZWja8bVc-Sok5roT_ja-uOISDYs0oBGE,4284
 ws_bom_robot_app/util.py,sha256=3aBK-bhsvKJwJeWOHh0c1B1BOyJ_tnUxOa1mJmFKwYQ,2618
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_description.py,sha256=9gK7MfsPvk7YTvk-bbLU_r4ygoEUUrepKxuj9zLxsgI,4681
 ws_bom_robot_app/llm/agent_handler.py,sha256=hbOf9i-ynDL3bcClqtUG-yWY8ohbCxONfT5ek9Cv0gY,5667
 ws_bom_robot_app/llm/agent_lcel.py,sha256=FzRt_u-zC78-DLS9cigO2oA6bsX6SOERuAFbevSXpT8,2701
-ws_bom_robot_app/llm/api.py,sha256=CyvBwyg6Wrxi0esMcTob8F3babuYJLr5huypLP0XpbQ,2695
+ws_bom_robot_app/llm/api.py,sha256=A1IFiiBLHdCJsFDllWTAdQEgkRm5a0BjU-oKozxlhiw,2880
 ws_bom_robot_app/llm/defaut_prompt.py,sha256=pn5a4lNLWE1NngHYjA_7tD8GasePMgsgude5fIJxsW0,756
 ws_bom_robot_app/llm/main.py,sha256=Q3A_Vgz09fhj78_PNVpDL9VQCYQd2lW_hmfyEKa_17Q,3667
 ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/models/api.py,sha256=Bi9YwNBJExoZzPVCerAoweWCY9zGAF9rn_N_aBNKmrU,6922
-ws_bom_robot_app/llm/models/kb.py,sha256=TqEAxMvXl0lyQhdojkRPXF_TW_zgtPf-uycGFyRXDsw,6612
+ws_bom_robot_app/llm/models/kb.py,sha256=EqC9M_nQzDKX9ttL5QG2d04aH19sh-svJdw1MinvGfk,8925
 ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/tools/tool_builder.py,sha256=rkYu0PrXV84PMi7INjCSWlrWMykUCI8aeF-QjZgLysM,854
 ws_bom_robot_app/llm/tools/tool_manager.py,sha256=y4K1NiDsVbdZjk2xUEK_T6j-4fHmV5QY02j5tHcoBRs,3708
@@ -23,18 +25,19 @@ ws_bom_robot_app/llm/tools/models/main.py,sha256=LsOJ7vkcSzYLoE1oa3TG0Rs0pr9J5VS
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent_utils.py,sha256=vdX_1Kxn6RcBljOSBlee1avi_0TyIk-rm0RGGeLt468,707
 ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBhttslIxqiI,3194
-ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=WJN1sljdLWHXISk7aabVsfu6uZJpE6B7JaPS8l6UQCg,4517
+ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=DowmroVT6eIbvnA-TG84PS_D7ujvxSRIKdLuIcJmd6Q,4650
+ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
 ws_bom_robot_app/llm/utils/print.py,sha256=bpLWY0KHXe7x7PWcWy8NS54ZWzHY8b4jrLRkpnDl108,818
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=yKIlSuaQKjnRrg8ut0_FZ0iF2zv8XCiQzawxpIsl7Qg,5210
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=_39HONBDSnIVOECoxyVHStg2XX4-m9Enm4pECm7wRBI,5816
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=EHGAF8W12XC6DIAFN-Qd3jCBAbq7yKxTk086cdg8e9E,625
+ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=eCKD3U0KPoVDMtKr2iZqauMFEKd9b2k6rqPG_YjDy0g,626
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=cSFlE2erMv3Uchy788mlCFdcvmyeoqdeIiGmJ9QbLhY,583
-ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=h2NhPv7k3p7wApUixq-zC4zMHz_4RLekigFqfst0bWg,3962
+ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=bdrgHBVB8Jt3xiRuomSIgJgnyEXoMJWdq4SvYPY_chA,4059
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=U9vxv7-CxLNGNAvdodcbhpOjaByxbXByt6EmZtA6Jjc,4487
-ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=2508f7RojpXCNd9hS2GC_wzXrVYsTiojsqrAJnxUvnA,829
-ws_bom_robot_app-0.0.5.dist-info/METADATA,sha256=_kTHKiXVLdx1uSil7npNBirDKFvIyK-uWB5UDyxwQps,5558
-ws_bom_robot_app-0.0.5.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.5.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.5.dist-info/RECORD,,
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=YKtJWTVQuOvRR2CLYTa0bGEbjqgmYVVud1YKQ9QLibc,4536
+ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=G9BoxwsevgqL72h2n28O2LpzaCYNymBkX66wru9GkCw,884
+ws_bom_robot_app-0.0.7.dist-info/METADATA,sha256=6CJ3Dxk0F_0N00bVKmu79zTkeP6n5EnyHkCUdBs0FKw,5608
+ws_bom_robot_app-0.0.7.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.7.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.7.dist-info/RECORD,,