ws-bom-robot-app 0.0.10__py3-none-any.whl → 0.0.12__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -1,6 +1,7 @@
+from typing import Optional
 from pydantic import BaseModel, ConfigDict
 from pydantic_settings import BaseSettings
-
+import os
 
 class Settings(BaseSettings):
     robot_env: str = 'local'
@@ -12,6 +13,7 @@ class Settings(BaseSettings):
     robot_data_db_folder_out: str = 'out'
     robot_data_db_folder_store: str = 'store'
     robot_data_db_retention_days: float = 60
+    robot_task_max_concurrent: int = os.cpu_count() or 1
     robot_task_retention_days: float = 1
     robot_cms_host: str = ''
     robot_cms_auth: str = ''
@@ -25,10 +27,37 @@ class Settings(BaseSettings):
     )
 
     class RuntimeOptions(BaseModel):
+        @staticmethod
+        def _get_number_of_workers() -> int:
+            """
+            Returns the number of worker processes to use for the application.
+
+            This function inspects the command-line arguments to determine the number
+            of worker processes to use. It looks for the "--workers" argument and
+            returns the subsequent value as an integer.
+            Sample of command-line arguments:
+            fastapi dev main.py --port 6001
+            fastapi run main.py --port 6001 --workers 4
+            uvicorn main:app --port 6001 --workers 4
+
+            Returns:
+                Optional[int]: The number of worker processes to use, or 1 if
+                               the argument is not found or the value is invalid.
+            """
+            import sys
+            try:
+                for i, arg in enumerate(sys.argv):
+                    if arg == "--workers" and i + 1 < len(sys.argv):
+                        return int(sys.argv[i + 1])
+            except (ValueError, IndexError):
+                pass
+            return 1
         debug: bool
         loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
+        number_of_workers: int = _get_number_of_workers()
+        is_multi_process: bool = _get_number_of_workers() > 1
 
 
     def runtime_options(self) -> RuntimeOptions:

ws_bom_robot_app/cron_manager.py

@@ -1,99 +1,251 @@
-from math import floor
-import schedule, time, threading
-import asyncio
+from apscheduler.schedulers.background import BackgroundScheduler
+#from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+from apscheduler.triggers.date import DateTrigger
 from fastapi import APIRouter
+from datetime import datetime
 from ws_bom_robot_app.task_manager import task_manager
 from ws_bom_robot_app.llm.utils.kb import kb_cleanup_data_file
 from ws_bom_robot_app.util import _log
-import random
-
-class RecurringJob():
-    @staticmethod
-    def __add_jitter(interval: int) -> int:
-      #delay proportional with interval, min 10, max 100 sec
-      jitter: int = max(10, min(100, floor(interval * 0.075)))
-      return interval + random.randint(-jitter, jitter)
-    def __init__(self, interval: int, job_func, tags: list[str]):
-        #add a little jitter by default for better concurrency in case of multiple instances
-        self.interval = RecurringJob.__add_jitter(interval)
-        self.job_func = job_func
-        self.is_coroutine = asyncio.iscoroutinefunction(job_func)
+from ws_bom_robot_app.config import config
+
+class JobstoreStrategy:
+    def get_jobstore(self):
+        raise NotImplementedError("Subclasses should implement this method")
+
+class MemoryJobstoreStrategy(JobstoreStrategy):
+    def get_jobstore(self):
+        _log.info("Using in-memory cron jobstore.")
+        return {"default": MemoryJobStore()}
+
+class PersistentJobstoreStrategy(JobstoreStrategy):
+    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
+        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+        return {"default": SQLAlchemyJobStore(url=db_url)}
+
+class Job:
+    def __init__(self, name: str, job_func, args: list = None, kwargs: dict = None, cron_expression: str = None, interval: int = None, run_at: datetime = None):
+        """
+        Job class that supports both recurring and one-time jobs.
+        :param job_func: The function to execute.
+        :param interval: Interval in seconds for recurring jobs.
+        :param run_at: Specific datetime for one-time jobs.
+        :param tags: Tags associated with the job.
+        """
+        if not (cron_expression or interval or run_at):
+            raise ValueError("Either 'interval' or 'run_at' must be provided.")
+        self.name = name
         self.job_func = job_func
-        self.tags = tags
-    def run(self):
-        if self.is_coroutine:
-            schedule.every(self.interval).seconds.do(self._run_async_job).tag(*self.tags)
-        else:
-            schedule.every(self.interval).seconds.do(self.job_func).tag(*self.tags)
-    async def _run_async_job(self):
-        await self.job_func()
+        self.args: list = args or []
+        self.kwargs: dict = kwargs or {}
+        self.cron_expression = cron_expression
+        self.interval = interval
+        self.run_at = run_at
+
+    def create_trigger(self):
+        """Create the appropriate trigger based on the job type."""
+        if self.cron_expression:
+            return CronTrigger.from_crontab(self.cron_expression)
+        if self.interval:
+            return IntervalTrigger(seconds=self.interval)
+        elif self.run_at:
+            return DateTrigger(run_date=self.run_at)
 
 class CronManager:
+    _list_default = [
+            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
+            Job('cleanup-data',kb_cleanup_data_file, interval=180 * 60),
+        ]
+    def __get_jobstore_strategy() -> JobstoreStrategy:
+        if True or config.runtime_options().is_multi_process:
+            return MemoryJobstoreStrategy()
+        return PersistentJobstoreStrategy()
+    def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):
+        self.enable_defaults = enable_defaults
+        if strategy is None:
+          strategy = CronManager.__get_jobstore_strategy()
+        jobstores = strategy.get_jobstore()
+        self.scheduler: BackgroundScheduler = BackgroundScheduler(jobstores=jobstores)
+        self.__scheduler_is_running = False
 
-    _list: dict[str, RecurringJob] = {
-      'cleanup-task': RecurringJob(5*60, task_manager.cleanup_task, tags=["cleanup","cleanup-task"]),
-      'cleanup-data': RecurringJob(180*60, kb_cleanup_data_file, tags=["cleanup","cleanup-data"]),
-    }
+    def add_job(self, job: Job):
+        """
+        Adds a job to the scheduler with the specified name and job details.
+        Args:
+          name (str): The unique identifier for the job.
+          job (Job): An instance of the Job class containing the job details.
+        The job details include:
+          - job_func: The function to be executed.
+          - args: The positional arguments to pass to the job function.
+          - kwargs: The keyword arguments to pass to the job function.
+          - trigger: The trigger that determines when the job should be executed.
+        The job will replace any existing job with the same name.
+        Sample usage:
+          recurring_job = Job(name="sample-recurring-job",job_func=example_job, interval=5, tags=tags, args=args, kwargs=kwargs)
+          cron_manager.add_job(recurring_job)
+          fire_once_job = Job(name="sample-fire-once-job",job_func=example_job, run_at=datetime.now(), tags=tags, args=args, kwargs=kwargs)
+          cron_manager.add_job(fire_once_job)
+        """
+        existing_job = self.scheduler.get_job(job.name)
+        if existing_job:
+            _log.info(f"Job with name '{job.name}' already exists. Skip creation.")
+        else:
+          trigger = job.create_trigger()
+          self.scheduler.add_job(
+              func=job.job_func,
+              args=job.args,
+              kwargs=job.kwargs,
+              trigger=trigger,
+              id=job.name,
+              name=job.name,
+              replace_existing=True
+          )
 
-    def __init__(self):
-        self.jobs: dict[str, RecurringJob] = CronManager._list
-        self.__scheduler_is_running = False
-    def add_job(self, name:str, job: RecurringJob):
-        job = {name: job}
-        self.jobs.append(job)
-        return job
-    def run_pending(self):
-        return schedule.run_pending()
-    def run_all(self):
-        return schedule.run_all()
-    def clear(self):
-        self.__scheduler_is_running = False
-        return schedule.clear()
-    def get_jobs(self):
-        return schedule.get_jobs()
     def start(self):
-        def _target():
-            while self.__scheduler_is_running:
-                time.sleep(1)
-                self.run_pending()
-                time.sleep(59)
-            _log.info(f"__scheduler_is_running={self.__scheduler_is_running}")
-        #clear all jobs
-        self.clear()
-        #prepare jobs
-        for job in self.jobs.values():
-            job.run()
-        #start scheduler
         if not self.__scheduler_is_running:
             self.__scheduler_is_running = True
-            t = threading.Thread(target=_target,args=(),daemon=True)
-            t.start()
+            self.scheduler.start()
+            if self.enable_defaults and CronManager._list_default:
+                for job in CronManager._list_default:
+                    existing_job = self.scheduler.get_job(job.name)
+                    if existing_job is None:
+                        self.add_job(job)
+
+
+    def get_job(self, job_id: str):
+        return self.scheduler.get_job(job_id)
+
+    def get_jobs(self):
+        return self.scheduler.get_jobs()
+
+    def execute_job(self, job_id: str):
+        job = self.scheduler.get_job(job_id)
+        if job:
+            job.func()
+        else:
+            raise ValueError(f"Job with id '{job_id}' not found.")
+
+    def pause_job(self, job_id: str):
+        self.scheduler.pause_job(job_id)
+
+    def resume_job(self, job_id: str):
+        self.scheduler.resume_job(job_id)
+
+    def remove_job(self, job_id: str):
+        self.scheduler.remove_job(job_id)
+
+    def execute_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                job.job_func()
+
+    def pause_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.pause_job(job.id)
+
+    def resume_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.resume_job(job.id)
+
+    def remove_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.remove_job(job.id)
+
+    def clear(self):
+        self.__scheduler_is_running = False
+        self.scheduler.remove_all_jobs()
+
+    def shutdown(self):
+        self.scheduler.shutdown()
 
 cron_manager = CronManager()
 
+# FastAPI Routes
 router = APIRouter(prefix="/api/cron", tags=["cron"])
+
 @router.get("/list")
 def _list():
-    def __format(job: schedule.Job) -> dict:
+    def __format(job):
+        return {
+            "id": job.id,
+            "name": job.name,
+            "func": job.func_ref,
+            "pending": job.pending,
+            "trigger": str(job.trigger),
+            "next_run_time": job.next_run_time
+        }
+    return [__format(job) for job in cron_manager.get_jobs()]
+
+@router.get("/default-jobs")
+def _default_jobs():
+    def __format(job):
+        existing_job = cron_manager.scheduler.get_job(job.name)
         return {
-            "job": {'module':job.job_func.__module__,'name':job.job_func.__name__},
-            "at": job.at_time,
-            "interval": job.interval,
-            "last_run": job.last_run,
-            "next_run": job.next_run,
-            "tags": job.tags}
-    _list = cron_manager.get_jobs()
-    return [__format(_) for _ in _list]
+            "name": job.name,
+            "status": "exists" if existing_job else "not added"
+        }
+    return [__format(job) for job in CronManager._list_default]
+
+@router.post("/execute-job/{job_id}")
+def _execute_job(job_id: str):
+    try:
+        cron_manager.execute_job(job_id)
+        return {"status": f"Job {job_id} executed"}
+    except ValueError as e:
+        return {"error": str(e)}
+
+@router.post("/pause-job/{job_id}")
+def _pause_job(job_id: str):
+    cron_manager.pause_job(job_id)
+    return {"status": f"Job {job_id} paused"}
+
+@router.post("/resume-job/{job_id}")
+def _resume_job(job_id: str):
+    cron_manager.resume_job(job_id)
+    return {"status": f"Job {job_id} resumed"}
+
+@router.delete("/remove-job/{job_id}")
+def _remove_job(job_id: str):
+    cron_manager.remove_job(job_id)
+    return {"status": f"Job {job_id} removed"}
+
+@router.post("/execute-recurring")
+def _execute_recurring():
+    cron_manager.execute_recurring_jobs()
+    return {"status": "All recurring jobs executed"}
+
+@router.post("/pause-recurring")
+def _pause_recurring():
+    cron_manager.pause_recurring_jobs()
+    return {"status": "All recurring jobs paused"}
+
+@router.post("/resume-recurring")
+def _resume_recurring():
+    cron_manager.resume_recurring_jobs()
+    return {"status": "All recurring jobs resumed"}
+
+@router.delete("/remove-recurring")
+def _remove_recurring():
+    cron_manager.remove_recurring_jobs()
+    return {"status": "All recurring jobs removed"}
 
 @router.get("/start")
 def _start():
     cron_manager.start()
+    return {"status": "started"}
+
 @router.delete("/stop")
 def _stop():
-    return {"_": cron_manager.clear()}
-@router.get("/run/pending")
-def _run_pending():
-    return {"_": cron_manager.run_pending()}
-@router.get("/run/all")
-def _run_all():
-    return {"_": cron_manager.run_all()}
+    cron_manager.clear()
+    return {"status": "stopped"}
+
+@router.get("/shutdown")
+def _shutdown():
+    cron_manager.shutdown()
+    return {"status": "shutdown"}

ws_bom_robot_app/llm/models/api.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 from datetime import datetime
 from pydantic import AliasChoices, BaseModel, Field, ConfigDict
 from ws_bom_robot_app.llm.models.kb import LlmKbEndpoint, LlmKbIntegration
@@ -125,7 +125,7 @@ class VectorDbRequest(BaseModel):
   def api_key(self):
     return self.secrets.get("openAIApiKey", "")
   def out_name(self):
-    return "vector_db_" + datetime.now().strftime("%Y-%m-%d_%H-%M-%S-%f")[:-3]
+    return f"db_{datetime.now().strftime("%Y-%m-%d_%H-%M-%S-%f")[:-3]}_{os.getpid()}"
 
 class RulesRequest(VectorDbRequest):
   type: Optional[str] = 'rules'

ws_bom_robot_app/llm/models/kb.py

@@ -144,7 +144,7 @@ async def load_endpoints(endpoints: list[LlmKbEndpoint], destination_directory:
             documents = await JsonLoader(
               file_path,
               meta_fields=[field.name for field in endpoint.fields_mapping.meta_fields] if endpoint.fields_mapping.meta_fields else []
-            ).load()
+            ).aload()
             _documents.extend(documents)
             await aiofiles.os.remove(file_path)
           except Exception as e:

ws_bom_robot_app/llm/utils/webhooks.py

@@ -51,3 +51,4 @@ class WebhookNotifier:
         async with httpx.AsyncClient(headers=_headers,verify=False,timeout=timeout) as client:
           response = await client.post(endpoint, data=_data)
           response.raise_for_status()
+

ws_bom_robot_app/llm/vector_store/generator.py

@@ -9,7 +9,7 @@ from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationMan
 from ws_bom_robot_app.llm.utils.faiss_helper import FaissHelper
 from ws_bom_robot_app.util import timer
 
-@timer
+#@timer
 async def rules(rq: RulesRequest) -> VectorDbResponse:
   api_key = rq.api_key()
   _config = rq.config()

ws_bom_robot_app/llm/vector_store/integration/sitemap.py

@@ -1,3 +1,4 @@
+from typing import Any
 import aiofiles
 import aiofiles.os
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
@@ -5,23 +6,21 @@ from langchain_community.document_loaders.sitemap import SitemapLoader
 from langchain_community.document_transformers import MarkdownifyTransformer as markdownify
 from langchain_core.documents import Document
 from bs4 import BeautifulSoup, Tag
-import nest_asyncio, os
-
 
 class Sitemap(IntegrationStrategy):
-    """_summary_
+    """Class to load a sitemap.xml file and extract text from the URLs.
       Load a sitemap.xml file and extract text from the urls.
     Args:
         data (dict[str, str]):
         data["sitemapUrl"] (str): absolute/relative url of the sitemap.xml
         data["outputFormat"] (str): ["text", "html", "markdown"] default to "text"
-        data["filterUrls"] list: list of regex pattern to filter urls ["https://www.example.com/en/products", "^.*products.*$"]
-        data["includeOnlySelector"] : [".content", "#main-article", "article p"]
-        data["excludeTag"] (str): default to ["script", "noscript", "style", "head", "header","nav","footer", "iframe"]
-        data["excludeClass"] (str): ["class1", "class2"]
-        data["excludeId"] (str): ["id1", "id2"]
+        data["filterUrls"] list[str]: list of regex pattern to filter urls ["https://www.example.com/en/products", "^.*products.*$"]
+        data["includeOnlySelector"] : list[str] [".content", "#main-article", "article p"]
+        data["excludeTag"] (list[str]): default to ["script", "noscript", "style", "head", "header","nav","footer", "iframe"]
+        data["excludeClass"] (list[str]): ["class1", "class2"]
+        data["excludeId"] (list[str]): ["id1", "id2"]
     """
-    def __init__(self, knowledgebase_path: str, data: dict[str, str]):
+    def __init__(self, knowledgebase_path: str, data: dict[str, Any]):
         super().__init__(knowledgebase_path, data)
         self.__sitemap_url = self.data.get("sitemapUrl")
         self.__filter_urls: list[str] = self.data.get("filterUrls",[]) # type: ignore
@@ -30,12 +29,12 @@ class Sitemap(IntegrationStrategy):
         self.__exclude_tag: list[str] = self.data.get("excludeTag",[]) # type: ignore
         self.__exclude_class: list[str] = self.data.get("excludeClass",[]) # type: ignore
         self.__exclude_id: list[str] = self.data.get("excludeId",[]) # type: ignore
-    def working_subdirectory(self) -> str: # type: ignore
+    def working_subdirectory(self) -> str:
         return ""
     def _extract(self, tag: Tag) -> str:
         return tag.get_text() if self.__output_format == "text" else tag.prettify()
     def _output(self, documents: list[Document]) -> list[Document]:
-        return list(markdownify().transform_documents(documents)) if (self.__output_format == "markdown") else documents
+        return list(markdownify().transform_documents(documents)) if self.__output_format == "markdown" else documents
     def _parse(self,content: BeautifulSoup) -> str:
         if self.__include_only_selectors:
             extracted = []
@@ -55,21 +54,25 @@ class Sitemap(IntegrationStrategy):
                 for _ in content.select(element):
                     _.decompose()
             return str(self._extract(content))
-    async def load(self) -> list[Document]:
-        def _is_local(url: str) -> bool:
-            return not url.startswith("http")
-        def _remap_if_local(url: str) -> str:
-            return f"{self.knowledgebase_path}/{url}" if _is_local(url) else url
+    def _is_local(self, url: str) -> bool:
+        return not url.startswith("http")
 
+    def _remap_if_local(self, url: str) -> str:
+        return f"{self.knowledgebase_path}/{url}" if self._is_local(url) else url
+
+    async def load(self) -> list[Document]:
         if (self.__sitemap_url):
             _loader = SitemapLoader(
-                web_path=_remap_if_local(self.__sitemap_url),
+                web_path=self._remap_if_local(self.__sitemap_url),
                 filter_urls=self.__filter_urls,
                 parsing_function=self._parse,
-                is_local=_is_local(self.__sitemap_url)
+                is_local=self._is_local(self.__sitemap_url)
             )
             _docs = self._output([document async for document in _loader.alazy_load()])
-            if _is_local(self.__sitemap_url):
-              await aiofiles.os.remove(_loader.web_path)
+            if self._is_local(self.__sitemap_url):
+              try:
+                  await aiofiles.os.remove(_loader.web_path)
+              except FileNotFoundError:
+                  pass
             return _docs
         return []

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -109,4 +109,5 @@ class Loader():
   #@timer
   async def load(self) -> list[Document]:
     loaders = MergedDataLoader(self.__directory_loader())
-    return await asyncio.to_thread(loaders.load)
+    return await loaders.aload()
+    #return await asyncio.to_thread(loaders.load)

ws_bom_robot_app/llm/vector_store/loader/json_loader.py

@@ -2,7 +2,6 @@ import json
 from typing import Optional
 from langchain_core.documents import Document
 from langchain_community.document_loaders.base import BaseLoader
-import aiofiles
 
 class JsonLoader(BaseLoader):
   def __init__(self, file_path: str, meta_fields:Optional[list[str]] = [],encoding: Optional[str] = "utf-8"):
@@ -10,9 +9,9 @@ class JsonLoader(BaseLoader):
     self.meta_fields = meta_fields
     self.encoding = encoding
 
-  async def load(self) -> list[Document]:
-    async with aiofiles.open(self.file_path, "r", encoding=self.encoding) as file:
-      data = json.loads(await file.read())
+  def load(self) -> list[Document]:
+    with open(self.file_path, "r", encoding=self.encoding) as file:
+      data = json.load(file)
     _list = data if isinstance(data, list) else [data]
     return [
       Document(

ws_bom_robot_app/main.py

@@ -1,5 +1,6 @@
 import datetime
 import platform
+from fastapi.responses import FileResponse
 import uvicorn, os, sys
 from fastapi import FastAPI, Depends
 from fastapi.openapi.docs import get_swagger_ui_html
@@ -22,7 +23,10 @@ app.include_router(cron,dependencies=[Depends(authenticate)])
 
 @app.get("/")
 async def root():
-    return {}
+    return health()
+@app.get("/favicon.ico")
+async def favicon():
+    return FileResponse("./favicon.ico")
 
 @app.get("/docs", include_in_schema=False)
 async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
@@ -31,7 +35,11 @@ async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
 async def openapi(authenticate: bool = Depends(authenticate)):
     return get_openapi(title=app.title, version=app.version, routes=app.routes)
 
-@app.get("/diag",tags=["diag"])
+@app.get("/api/health",tags=["diag"])
+def health():
+    return {"status": "ok"}
+
+@app.get("/api/diag",tags=["diag"])
 def diag(authenticate: bool = Depends(authenticate)):
     import pkg_resources
     from ws_bom_robot_app.llm.vector_store.loader.base import Loader as wsll
@@ -48,6 +56,7 @@ def diag(authenticate: bool = Depends(authenticate)):
                 "version": platform.version(),
                 "type": platform.machine(),
                 "processor": platform.processor(),
+                "cpu": os.cpu_count(),
                 "architecture": platform.architecture()
             },
             "sys": {
@@ -57,6 +66,7 @@ def diag(authenticate: bool = Depends(authenticate)):
                 "args": {k: arg for k, arg in enumerate(sys.argv)}
             },
             "os": {
+                "ppid": os.getppid(),
                 "pid": os.getpid(),
                 "cwd": os.getcwd(),
                 "ws_bom_robot_app": pkg_resources.get_distribution("ws_bom_robot_app").version,
@@ -64,6 +74,7 @@ def diag(authenticate: bool = Depends(authenticate)):
             },
         },
         "config":config,
+        "runtime":config.runtime_options(),
         "extension": {
             "loader": ({item[0]: item[1].loader.__name__ if item[1] else None} for item in sorted(wsll._list.items(), key=lambda x: x[0]) if item[1]),
             "integration":({item[0]: type(item[1]).__name__} for item in wsim._list.items()),

ws_bom_robot_app/task_manager.py

@@ -1,53 +1,78 @@
+from collections import deque
 import inspect
+from math import floor
 import asyncio, os
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Annotated, TypeVar, Optional, Dict, Union, Any
+from typing import Annotated, Coroutine, Literal, TypeVar, Optional, Dict, Union, Any
 from pydantic import BaseModel, ConfigDict, Field, computed_field
 from uuid import uuid4
 from fastapi import APIRouter, HTTPException
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.models.base import IdentifiableEntity
 from ws_bom_robot_app.llm.utils.webhooks import WebhookNotifier
+from ws_bom_robot_app.util import _log
+from sqlalchemy import create_engine, Column, String, JSON, DateTime, Enum
+from sqlalchemy.orm import sessionmaker, registry
+from abc import ABC, abstractmethod
 
 T = TypeVar('T')
 
+#region models
 class TaskHeader(BaseModel):
+    """
+    TaskHeader model representing the header information for a task.
+    Example:
+    ```bash
+    curl -X POST "http://localhost:6001/api/llm/kb/task"
+      -H  "x-ws-bom-msg-id: 1234"
+      -H  "x-ws-bom-msg-type: generate.knowledgebase"
+      -H  "x-ws-bom-msg-extra: key1=value1,key2=value2"
+      -H  "x-ws-bom-webhooks: http://localhost:8000/api/webhook"
+      -d "{\"api_key\":\"string\"}"
+    ```
+    Attributes:
+      x_ws_bom_msg_id (Optional[str]): The message ID for the task. If not provided, a UUID will be generated.
+      x_ws_bom_msg_type (Optional[str]): The message type for the task, e.g. "send.email" or "generate.knowledgebase".
+      x_ws_bom_msg_extra (Optional[str]): Any extra information for the task, in comma separated key=value pairs. e.g. "key1=value1,key2=value2".
+      x_ws_bom_webhooks (Optional[str]): Webhooks associated with the task, called when the task is completed or failed.
+    """
+    x_ws_bom_msg_id: Optional[str] = None
     x_ws_bom_msg_type: Optional[str] = None
+    x_ws_bom_msg_extra: Optional[str] = None
     x_ws_bom_webhooks: Optional[str] = None
     model_config = ConfigDict(
         extra='allow'
     )
 
 class TaskMetaData(BaseModel):
-    start_time: datetime
-    end_time: Optional[datetime] = None
+    created_at: str
+    start_at: Optional[str] = None
+    end_at: Optional[str] = None
     @computed_field
     @property
-    def elapsed_time(self) -> Union[timedelta, None]:
-        return (datetime.now() if not self.end_time else self.end_time) - self.start_time
+    def elapsed_time(self) -> Union[str, None]:
+        return str(
+            (datetime.now() if not self.end_at else datetime.fromisoformat(self.end_at))
+            - datetime.fromisoformat(self.created_at if not self.start_at else self.start_at)
+            )
     source: Optional[str] = None
     pid: Optional[int] = None
-    model_config = ConfigDict(
-        json_encoders={
-            datetime: lambda v: v.isoformat(),
-            timedelta: lambda v: str(v)
-        }
-    )
+    extra: Optional[dict[str,str]] = None
 
 class TaskStatus(IdentifiableEntity):
-    class TaskStatusEnum(str, Enum):
-        pending = "pending"
-        completed = "completed"
-        failure = "failure"
     type: Optional[str] = None
-    status: TaskStatusEnum
+    status: Literal["pending", "completed", "failure"]
     result: Optional[T] = None
     metadata: TaskMetaData = None
     error: Optional[str] = None
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
 
 class TaskEntry(IdentifiableEntity):
     task: Annotated[asyncio.Task, Field(default=None, validate_default=False)] = None
+    coroutine: Coroutine = None
     headers: TaskHeader | None = None
     status: Union[TaskStatus, None] = None
     def _get_coroutine_name(self, coroutine: asyncio.coroutines) -> str:
@@ -55,6 +80,15 @@ class TaskEntry(IdentifiableEntity):
             return coroutine.cr_code.co_name
         return "<unknown>"
     def __init__(self, **data):
+        def _metadata_extra(data: str) -> dict[str,str] | None:
+            if data:
+                _values = data.split(",")
+                if _values:
+                    try:
+                        return {k: v for k,v in [val.split("=") for val in _values]}
+                    except Exception as e:
+                        return None
+            return None
         #separate task from data to handle asyncio.Task
         task = data.pop('task',None)
         super().__init__(**data)
@@ -64,88 +98,290 @@ class TaskEntry(IdentifiableEntity):
         if not self.status:
           self.status = TaskStatus(
               id=self.id,
-              type=self.headers.x_ws_bom_msg_type if self.headers and self.headers.x_ws_bom_msg_type else self._get_coroutine_name(task._coro) if task else None,
-              status=TaskStatus.TaskStatusEnum.pending,
+              type=self.headers.x_ws_bom_msg_type if self.headers and self.headers.x_ws_bom_msg_type else self._get_coroutine_name(self.coroutine) if self.coroutine else None,
+              status="pending",
               metadata=TaskMetaData(
-                 start_time=datetime.now(),
-                 source=self._get_coroutine_name(task._coro) if task else None,
-                 pid=os.getpid())
+                 created_at=str(datetime.now().isoformat()),
+                 source=self._get_coroutine_name(self.coroutine) if self.coroutine else None,
+                 pid=os.getpid(),
+                 extra=_metadata_extra(self.headers.x_ws_bom_msg_extra) if self.headers and self.headers.x_ws_bom_msg_extra else None
+                 )
               )
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
         validate_assignment=True
     )
 
-class TaskManager:
-    def __init__(self):
-        self.tasks: Dict[str, TaskEntry] = {}
+class TaskStatistics(BaseModel):
+    class TaskStatisticExecutionInfo(BaseModel):
+        retention_days: float = config.robot_task_retention_days
+        max_concurrent: int
+        running: list[TaskStatus]
+        slowest: list
+    class TaskStatisticExecutionTime(BaseModel):
+        min: str
+        max: str
+        avg: str
+    total: int
+    pending: int
+    completed: int
+    failure: int
+    exec_time: TaskStatisticExecutionTime
+    exec_info: TaskStatisticExecutionInfo
+
+#endregion
+
+#region interface
+class TaskManagerStrategy(ABC):
+    def __init__(self, max_concurrent_tasks: int = floor(2 * config.robot_task_max_concurrent / config.runtime_options().number_of_workers)):
+        self.max_concurrent_tasks = max_concurrent_tasks
+        self.semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
+        self.running_tasks = dict[str, TaskEntry]()
+        self.loop = asyncio.get_event_loop()
+
+    @abstractmethod
+    def create_task(self, coroutine, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        pass
+
+    @abstractmethod
+    def update_task_status(self, task: TaskEntry) -> None:
+        """Hook for additional behavior, such as persisting the task status."""
+        pass
+
+    @abstractmethod
+    def get_task(self, id: str) -> TaskStatus | None:
+        pass
+
+    @abstractmethod
+    def get_tasks(self) -> list[TaskStatus]:
+        pass
 
-    def _task_done_callback(self, task_id: str, headers: TaskHeader | None = None):
+    @abstractmethod
+    def remove_task(self, id: str) -> None:
+        pass
+
+    @abstractmethod
+    def cleanup_task(self) -> None:
+        pass
+
+    @abstractmethod
+    def stats(self) -> TaskStatistics:
+        pass
+
+    def task_cleanup_rule(self, task: TaskEntry) -> bool:
+        return task.status.status in {"completed", "failure"} and datetime.fromisoformat(task.status.metadata.end_at) < datetime.now() - timedelta(days=config.robot_task_retention_days)
+
+    def task_done_callback(self, task_entry: TaskEntry, headers: TaskHeader | None = None) -> callable:
         def callback(task: asyncio.Task):
-            if _task := self.tasks.get(task_id):
-              try:
-                  result = _task.task.result()
-                  _task.status.status = TaskStatus.TaskStatusEnum.completed
-                  _task.status.result = result
-              except Exception as e:
-                  _task.status.status = TaskStatus.TaskStatusEnum.failure
-                  _task.status.error = str(e)
-              finally:
-                  _task.status.metadata.end_time = datetime.now()
-                  if headers and headers.x_ws_bom_webhooks:
-                        asyncio.create_task(
-                            WebhookNotifier().notify_webhook(_task.status,headers.x_ws_bom_webhooks)
-                            )
+            try:
+                result = task_entry.task.result()
+                task_entry.status.status = "completed"
+                task_entry.status.result = result
+            except Exception as e:
+                task_entry.status.status = "failure"
+                task_entry.status.error = str(e)
+            finally:
+                task_entry.status.metadata.end_at = str(datetime.now().isoformat())
+                #strategy-specific behavior
+                self.update_task_status(task_entry)
+                #notify webhooks
+                if headers and headers.x_ws_bom_webhooks:
+                      asyncio.create_task(
+                          WebhookNotifier().notify_webhook(task_entry.status,headers.x_ws_bom_webhooks)
+                          )
         return callback
 
-    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
-        _task = asyncio.create_task(coroutine)
+    def create_task_entry(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> TaskEntry:
+        _id = headers and headers.x_ws_bom_msg_id or str(uuid4())
         task = TaskEntry(
-            id=str(uuid4()),
-            task=_task,
+            id=_id,
+            coroutine=coroutine,
             headers=headers)
-        task.task.add_done_callback(self._task_done_callback(task.id, headers))
+        self.loop.create_task(self._run_task_with_semaphore(task)) # run the task
+        return task
+
+    async def _run_task_with_semaphore(self, task_entry: TaskEntry):
+        """Run a task with semaphore control to limit concurrency."""
+        async with self.semaphore:
+          self.running_tasks[task_entry.id]=task_entry
+          await self._execute_task(task_entry)
+          del self.running_tasks[task_entry.id]
+
+    async def _execute_task(self, task_entry: TaskEntry):
+        """Execute a task and handle its lifecycle."""
+        task_entry.status.metadata.start_at = str(datetime.now().isoformat())
+        task_entry.task = asyncio.create_task(task_entry.coroutine)
+        task_entry.task.add_done_callback(self.task_done_callback(task_entry, task_entry.headers))
+        await task_entry.task
+
+    def running_task(self):
+        return self.running_tasks.values()
+    def stats(self) -> TaskStatistics:
+        def __string_to_timedelta(value: str) -> timedelta:
+            if "." in value:
+                time_format = "%H:%M:%S.%f"
+            else:
+                time_format = "%H:%M:%S"
+            time_obj = datetime.strptime(value, time_format)
+            return timedelta(hours=time_obj.hour, minutes=time_obj.minute, seconds=time_obj.second, microseconds=time_obj.microsecond)
+        def __timedelta_to_string(td):
+            hours, remainder = divmod(td.total_seconds(), 3600)
+            minutes, seconds = divmod(remainder, 60)
+            return f"{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d}.{td.microseconds}"
+        _all = self.get_tasks()
+        _not_pending = _all and [task for task in _all if task.status != "pending"] or []
+        _total_not_pending = _not_pending and len(_not_pending) if _not_pending else 0
+        elapsed_times = _not_pending and [__string_to_timedelta(task.metadata.elapsed_time) for task in _not_pending]
+        _avg_exec_time = sum(elapsed_times, timedelta()) / _total_not_pending if elapsed_times and _total_not_pending > 0 else timedelta()
+        _min_exec_time = min(elapsed_times) if elapsed_times and _total_not_pending > 0 else timedelta()
+        _max_exec_time = max(elapsed_times) if elapsed_times and _total_not_pending > 0 else timedelta()
+        _slowest: list[TaskStatus] = _not_pending and sorted(_not_pending, key=lambda x: __string_to_timedelta(x.metadata.elapsed_time), reverse=True)[:3]
+        return TaskStatistics(
+            total= _all and len(_all) or 0,
+            pending=_all and len([task for task in _all if task.status == "pending"]) or 0,
+            completed=_all and len([task for task in _all if task.status == "completed"]) or 0,
+            failure=_all and len([task for task in _all if task.status == "failure"]) or 0,
+            exec_time=TaskStatistics.TaskStatisticExecutionTime(
+                min=__timedelta_to_string(_min_exec_time),
+                max=__timedelta_to_string(_max_exec_time),
+                avg=__timedelta_to_string(_avg_exec_time)
+            ),
+            exec_info=TaskStatistics.TaskStatisticExecutionInfo(
+                retention_days=config.robot_task_retention_days,
+                max_concurrent=self.max_concurrent_tasks,
+                running=[task.status for task in self.running_task()],
+                slowest=_slowest
+            )
+        )
+
+#endregion
+
+#memory implementation
+class MemoryTaskManagerStrategy(TaskManagerStrategy):
+    def __init__(self):
+        super().__init__()
+        self.tasks: Dict[str, TaskEntry] = {}
+
+    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        task = self.create_task_entry(coroutine, headers)
         self.tasks[task.id] = task
         return IdentifiableEntity(id=task.id)
 
-    def get_task(self, task_id: str) -> TaskEntry | None:
-        if _task := self.tasks.get(task_id):
-            return _task
+    def update_task_status(self, task: TaskEntry) -> None:
+        """no-op for memory strategy."""
+        pass
+
+    def get_task(self, id: str) -> TaskStatus | None:
+        if _task := self.tasks.get(id):
+            return _task.status
+        return None
+
+    def get_tasks(self) -> list[TaskStatus] | None:
+        return [task.status for task in self.tasks.values()]
+
+    def remove_task(self, id: str) -> None:
+        if id in self.tasks:
+            del self.tasks[id]
+
+    def cleanup_task(self):
+        keys = [task.id for task in self.tasks.values() if self.task_cleanup_rule(task)]
+        for key in keys:
+            self.remove_task(key)
+
+#endregion
+
+#db implementation
+Base = registry().generate_base()
+class TaskEntryModel(Base):
+    __tablename__ = "entry"
+    id = Column(String, primary_key=True)
+    status = Column(JSON)
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
+class DatabaseTaskManagerStrategy(TaskManagerStrategy):
+    def __init__(self, db_url: str = "sqlite:///.data/db/tasks.sqlite"):
+        super().__init__()
+        self.engine = create_engine(db_url)
+        self.Session = sessionmaker(bind=self.engine)
+        Base.metadata.create_all(self.engine)
+
+    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        task = self.create_task_entry(coroutine, headers)
+        with self.Session() as session:
+            session.add(TaskEntryModel(id=task.id, status=task.status.model_dump()))
+            session.commit()
+        return IdentifiableEntity(id=task.id)
+
+    def update_task_status(self, task: TaskEntry) -> None:
+        with self.Session() as session:
+          session.query(TaskEntryModel).filter_by(id=task.id).update(
+              {"status": task.status.model_dump()}
+          )
+          session.commit()
+
+    def get_task(self, id: str) -> TaskStatus | None:
+        with self.Session() as session:
+            task = session.query(TaskEntryModel).filter_by(id=id).first()
+            if task:
+                return TaskEntry(**task.__dict__).status
         return None
 
-    def remove_task(self, task_id: str) -> None:
-        if task_id in self.tasks:
-            del self.tasks[task_id]
+    def get_tasks(self) -> list[TaskStatus]:
+        with self.Session() as session:
+            tasks = session.query(TaskEntryModel).all()
+            if tasks:
+                return [TaskEntry(**task.__dict__).status for task in tasks]
+        return []
+
+    def remove_task(self, id: str) -> None:
+        with self.Session() as session:
+            session.query(TaskEntryModel).filter_by(id=id).delete()
+            session.commit()
 
     def cleanup_task(self):
-        for task_id in [task_id for task_id, task in self.tasks.items()
-                        if task.status.status in {TaskStatus.TaskStatusEnum.completed, TaskStatus.TaskStatusEnum.failure}
-                        and task.status.metadata.end_time < datetime.now() - timedelta(days=config.robot_task_retention_days)]:
-            self.remove_task(task_id)
+        with self.Session() as session:
+            for task in session.query(TaskEntryModel).all():
+                _task = TaskEntry(**task.__dict__)
+                if self.task_cleanup_rule(_task):
+                    session.query(TaskEntryModel).filter_by(id=task.id).delete()
+            session.commit()
+#endregion
 
 # global instance
-task_manager = TaskManager()
+def __get_taskmanager_strategy() -> TaskManagerStrategy:
+    if config.runtime_options().is_multi_process:
+        return DatabaseTaskManagerStrategy()
+    return MemoryTaskManagerStrategy()
+task_manager = __get_taskmanager_strategy()
+_log.info(f"Task manager strategy: {task_manager.__class__.__name__}")
 
+#region api
 router = APIRouter(prefix="/api/task", tags=["task"])
-@router.get("/status/{task_id}", response_model=TaskStatus)
-async def _status_task(task_id: str) -> TaskStatus:
-    task = task_manager.get_task(task_id)
-    if not task:
+
+@router.get("/status/{id}")
+async def _status_task(id: str) -> TaskStatus:
+    task_status = task_manager.get_task(id)
+    if not task_status:
         raise HTTPException(status_code=404, detail="Task not found")
-    return task.status
+    return task_status
+
 @router.get("/status")
-async def _status_task_list():
-    _status_task_list = []
-    for task in task_manager.tasks.values():
-        _task = task_manager.get_task(task.id)
-        _status_task_list.append(_task.status)
-    return _status_task_list
-@router.delete("/status/{task_id}")
-async def _remove_task(task_id: str):
-    task_manager.remove_task(task_id)
-    return {"success":"ok"}
+async def _status_task_list() -> list[TaskStatus]:
+    return task_manager.get_tasks()
+
+@router.delete("/status/{id}")
+async def _remove_task(id: str):
+   task_manager.remove_task(id)
+   return {"success":"ok"}
+
 @router.delete("/cleanup")
 async def _remove_task_list():
     task_manager.cleanup_task()
     return {"success":"ok"}
 
+@router.get("/stats")
+async def _stats() -> TaskStatistics:
+    return task_manager.stats()
+
+#endregion

{ws_bom_robot_app-0.0.10.dist-info → ws_bom_robot_app-0.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.10
+Version: 0.0.12
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -11,7 +11,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
-Requires-Dist: schedule==1.2.2
+Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: pydantic==2.9.2
 Requires-Dist: pydantic-settings==2.6.0

{ws_bom_robot_app-0.0.10.dist-info → ws_bom_robot_app-0.0.12.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
-ws_bom_robot_app/config.py,sha256=VUoE6MBpYnZ8FQvUjWOLCzfs35aUvpwrz4ROFVbO3AA,1831
-ws_bom_robot_app/cron_manager.py,sha256=Gd76VwhvlauZ5jCrZa4gyM1NoQdynzskfRYPw6niFyk,3548
-ws_bom_robot_app/main.py,sha256=UPRqJkdhFBS8vmJYFJb4hgpx4pMFwOCrbJwt5MTQymE,3556
-ws_bom_robot_app/task_manager.py,sha256=aiQV5EJ_Q0B0mTcERVbzgaEepEUf4OtljC_UACs6pr8,5837
+ws_bom_robot_app/config.py,sha256=6elpogZZW8QXD7oJX4DhYTLqsVFHq2wMcR1fz1X4qdo,3157
+ws_bom_robot_app/cron_manager.py,sha256=0Yt5AMTPGlXZ_M5ck0SKMX8wvzoPsseEezg_s0Q3HKY,9224
+ws_bom_robot_app/main.py,sha256=MIR2WgxX9HwkNSY2JMRRxrLt-ZGs_TfrhX_BbTbigTI,3909
+ws_bom_robot_app/task_manager.py,sha256=7rPFvYzaVL4DSKjLXCX5SXA3mN5qJ7OwfLS0-hvvRwE,15885
 ws_bom_robot_app/util.py,sha256=3aBK-bhsvKJwJeWOHh0c1B1BOyJ_tnUxOa1mJmFKwYQ,2618
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_description.py,sha256=SDJYMmwfdMxEK3a_HDEQ19bfNKmwMSFf5hqU0VSCCIE,4705
@@ -14,9 +14,9 @@ ws_bom_robot_app/llm/defaut_prompt.py,sha256=pn5a4lNLWE1NngHYjA_7tD8GasePMgsgude
 ws_bom_robot_app/llm/main.py,sha256=HWTaAmxZpVo9HEc2cfv9DqA3tlaVGl1NdBONgEI16ds,3595
 ws_bom_robot_app/llm/settings.py,sha256=EkFGCppORenStH9W4e6_dYvQ-5p6xiEMpmUHBqNqG9M,117
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/models/api.py,sha256=IGeHJhyMaZ5iqOPNj6Bko2jp6R9yALMfpyeOzwW_NGk,6982
+ws_bom_robot_app/llm/models/api.py,sha256=KlVUbApyz6uuWefAN9K4B_vWDSps5hLW6hNg1Eo3TBQ,6996
 ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
-ws_bom_robot_app/llm/models/kb.py,sha256=wsvIb1aaZO0M-04VsKY-VuplHSHyft6UgYOPEjjDAnc,8934
+ws_bom_robot_app/llm/models/kb.py,sha256=9zqwDlVULVrWE48wo5AivzWoOtnjA57k9rsw8KNnyDk,8935
 ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/tools/tool_builder.py,sha256=rkYu0PrXV84PMi7INjCSWlrWMykUCI8aeF-QjZgLysM,854
 ws_bom_robot_app/llm/tools/tool_manager.py,sha256=y4K1NiDsVbdZjk2xUEK_T6j-4fHmV5QY02j5tHcoBRs,3708
@@ -29,17 +29,17 @@ ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBh
 ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=DowmroVT6eIbvnA-TG84PS_D7ujvxSRIKdLuIcJmd6Q,4650
 ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
 ws_bom_robot_app/llm/utils/print.py,sha256=bpLWY0KHXe7x7PWcWy8NS54ZWzHY8b4jrLRkpnDl108,818
-ws_bom_robot_app/llm/utils/webhooks.py,sha256=Sdwu4dkbUC3JEf9ck1hsu4gSGQdj7gcsIrhs3cOVXxk,2112
+ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=DM1jQB6nb03GYFMaO-qN6IX29apFZOYkxEKt6wrXdIQ,5770
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=SrxrZ87JmWW4PQ-zP8upJJfamWur49fvH2eoIjEVoCI,5771
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=eCKD3U0KPoVDMtKr2iZqauMFEKd9b2k6rqPG_YjDy0g,626
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=cSFlE2erMv3Uchy788mlCFdcvmyeoqdeIiGmJ9QbLhY,583
-ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=O7E6FfE0gUg6kMkJUy8t19ZnJ74zm_nusNrWWLz_zjo,4031
+ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=nPbIywp-ZwWbWStvjvYVgHqqejyYFr8eZhBc8ycTuaU,4206
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=YKtJWTVQuOvRR2CLYTa0bGEbjqgmYVVud1YKQ9QLibc,4536
-ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=G9BoxwsevgqL72h2n28O2LpzaCYNymBkX66wru9GkCw,884
-ws_bom_robot_app-0.0.10.dist-info/METADATA,sha256=OfGHsmNN39wqXEptnM_p1en48OhEKQHN0KvCN1SsBp4,5674
-ws_bom_robot_app-0.0.10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.10.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.10.dist-info/RECORD,,
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=dhZ7F4EJmuYa2TBMggWVpQe4_NmS2wi312lHnNm5Jm0,4571
+ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
+ws_bom_robot_app-0.0.12.dist-info/METADATA,sha256=2ATPWZDFjdg9vQw3MLBJOXaGUU8f1JgrrlhAOGKQpyw,5678
+ws_bom_robot_app-0.0.12.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.12.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.12.dist-info/RECORD,,