ws-bom-robot-app 0.0.10__py3-none-any.whl → 0.0.11__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from pydantic import BaseModel, ConfigDict
 from pydantic_settings import BaseSettings
 
@@ -25,10 +26,38 @@ class Settings(BaseSettings):
     )
 
     class RuntimeOptions(BaseModel):
+        def _is_multi_process() -> bool:
+            """
+            Checks if the application is running with multiple worker processes.
+
+            This function inspects the command-line arguments to determine if the
+            application is configured to run with more than one worker process. It
+            looks for the "--workers" argument and checks if the subsequent value
+            is greater than 1.
+            Sample of command-line arguments:
+            fastapi run main.py --port 6001 --workers 4
+            uvicorn main:app --port 6001 --workers 4
+
+            Returns:
+                bool: True if the application is running with multiple worker
+                      processes, False otherwise.
+            """
+            import sys, os
+            try:
+                for i, arg in enumerate(sys.argv):
+                    if arg == "--workers" and i + 1 < len(sys.argv):
+                        workers = int(sys.argv[i + 1])
+                        if workers > 1:
+                            return True
+            except (ValueError, IndexError):
+                pass
+            # Fallback: Compare process and parent process IDs
+            return False #os.getpid() != os.getppid()
         debug: bool
         loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
+        is_multi_process: bool = _is_multi_process()
 
 
     def runtime_options(self) -> RuntimeOptions:

ws_bom_robot_app/cron_manager.py

@@ -1,99 +1,250 @@
-from math import floor
-import schedule, time, threading
-import asyncio
+from apscheduler.schedulers.background import BackgroundScheduler
+#from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+from apscheduler.triggers.date import DateTrigger
 from fastapi import APIRouter
+from datetime import datetime
 from ws_bom_robot_app.task_manager import task_manager
 from ws_bom_robot_app.llm.utils.kb import kb_cleanup_data_file
 from ws_bom_robot_app.util import _log
-import random
-
-class RecurringJob():
-    @staticmethod
-    def __add_jitter(interval: int) -> int:
-      #delay proportional with interval, min 10, max 100 sec
-      jitter: int = max(10, min(100, floor(interval * 0.075)))
-      return interval + random.randint(-jitter, jitter)
-    def __init__(self, interval: int, job_func, tags: list[str]):
-        #add a little jitter by default for better concurrency in case of multiple instances
-        self.interval = RecurringJob.__add_jitter(interval)
-        self.job_func = job_func
-        self.is_coroutine = asyncio.iscoroutinefunction(job_func)
+from ws_bom_robot_app.config import config
+
+class JobstoreStrategy:
+    def get_jobstore(self):
+        raise NotImplementedError("Subclasses should implement this method")
+
+class MemoryJobstoreStrategy(JobstoreStrategy):
+    def get_jobstore(self):
+        _log.info("Using in-memory cron jobstore.")
+        return {"default": MemoryJobStore()}
+
+class PersistentJobstoreStrategy(JobstoreStrategy):
+    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
+        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+        return {"default": SQLAlchemyJobStore(url=db_url)}
+
+class Job:
+    def __init__(self, name: str, job_func, args: list = None, kwargs: dict = None, cron_expression: str = None, interval: int = None, run_at: datetime = None):
+        """
+        Job class that supports both recurring and one-time jobs.
+        :param job_func: The function to execute.
+        :param interval: Interval in seconds for recurring jobs.
+        :param run_at: Specific datetime for one-time jobs.
+        :param tags: Tags associated with the job.
+        """
+        if not (cron_expression or interval or run_at):
+            raise ValueError("Either 'interval' or 'run_at' must be provided.")
+        self.name = name
         self.job_func = job_func
-        self.tags = tags
-    def run(self):
-        if self.is_coroutine:
-            schedule.every(self.interval).seconds.do(self._run_async_job).tag(*self.tags)
-        else:
-            schedule.every(self.interval).seconds.do(self.job_func).tag(*self.tags)
-    async def _run_async_job(self):
-        await self.job_func()
+        self.args: list = args or []
+        self.kwargs: dict = kwargs or {}
+        self.cron_expression = cron_expression
+        self.interval = interval
+        self.run_at = run_at
+
+    def create_trigger(self):
+        """Create the appropriate trigger based on the job type."""
+        if self.cron_expression:
+            return CronTrigger.from_crontab(self.cron_expression)
+        if self.interval:
+            return IntervalTrigger(seconds=self.interval)
+        elif self.run_at:
+            return DateTrigger(run_date=self.run_at)
 
 class CronManager:
+    _list_default = [
+            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
+            Job('cleanup-data',kb_cleanup_data_file, interval=180 * 60),
+        ]
+    def __get_jobstore_strategy() -> JobstoreStrategy:
+        if config.runtime_options().is_multi_process:
+            return MemoryJobstoreStrategy()
+        return PersistentJobstoreStrategy()
+    def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):
+        self.enable_defaults = enable_defaults
+        if strategy is None:
+          strategy = CronManager.__get_jobstore_strategy()
+        jobstores = strategy.get_jobstore()
+        self.scheduler: BackgroundScheduler = BackgroundScheduler(jobstores=jobstores)
+        self.__scheduler_is_running = False
 
-    _list: dict[str, RecurringJob] = {
-      'cleanup-task': RecurringJob(5*60, task_manager.cleanup_task, tags=["cleanup","cleanup-task"]),
-      'cleanup-data': RecurringJob(180*60, kb_cleanup_data_file, tags=["cleanup","cleanup-data"]),
-    }
+    def add_job(self, job: Job):
+        """
+        Adds a job to the scheduler with the specified name and job details.
+        Args:
+          name (str): The unique identifier for the job.
+          job (Job): An instance of the Job class containing the job details.
+        The job details include:
+          - job_func: The function to be executed.
+          - args: The positional arguments to pass to the job function.
+          - kwargs: The keyword arguments to pass to the job function.
+          - trigger: The trigger that determines when the job should be executed.
+        The job will replace any existing job with the same name.
+        Sample usage:
+          recurring_job = Job(name="sample-recurring-job",job_func=example_job, interval=5, tags=tags, args=args, kwargs=kwargs)
+          cron_manager.add_job(recurring_job)
+          fire_once_job = Job(name="sample-fire-once-job",job_func=example_job, run_at=datetime.now(), tags=tags, args=args, kwargs=kwargs)
+          cron_manager.add_job(fire_once_job)
+        """
+        existing_job = self.scheduler.get_job(job.name)
+        if existing_job:
+            _log.info(f"Job with name '{job.name}' already exists. Skip creation.")
+        else:
+          trigger = job.create_trigger()
+          self.scheduler.add_job(
+              func=job.job_func,
+              args=job.args,
+              kwargs=job.kwargs,
+              trigger=trigger,
+              id=job.name,
+              name=job.name,
+              replace_existing=False
+          )
 
-    def __init__(self):
-        self.jobs: dict[str, RecurringJob] = CronManager._list
-        self.__scheduler_is_running = False
-    def add_job(self, name:str, job: RecurringJob):
-        job = {name: job}
-        self.jobs.append(job)
-        return job
-    def run_pending(self):
-        return schedule.run_pending()
-    def run_all(self):
-        return schedule.run_all()
-    def clear(self):
-        self.__scheduler_is_running = False
-        return schedule.clear()
-    def get_jobs(self):
-        return schedule.get_jobs()
     def start(self):
-        def _target():
-            while self.__scheduler_is_running:
-                time.sleep(1)
-                self.run_pending()
-                time.sleep(59)
-            _log.info(f"__scheduler_is_running={self.__scheduler_is_running}")
-        #clear all jobs
-        self.clear()
-        #prepare jobs
-        for job in self.jobs.values():
-            job.run()
-        #start scheduler
         if not self.__scheduler_is_running:
             self.__scheduler_is_running = True
-            t = threading.Thread(target=_target,args=(),daemon=True)
-            t.start()
+            self.scheduler.start()
+            if self.enable_defaults and CronManager._list_default:
+                for job in CronManager._list_default:
+                    existing_job = self.scheduler.get_job(job.name)
+                    if existing_job is None:
+                        self.add_job(job)
+
+    def get_job(self, job_id: str):
+        return self.scheduler.get_job(job_id)
+
+    def get_jobs(self):
+        return self.scheduler.get_jobs()
+
+    def execute_job(self, job_id: str):
+        job = self.scheduler.get_job(job_id)
+        if job:
+            job.func()
+        else:
+            raise ValueError(f"Job with id '{job_id}' not found.")
+
+    def pause_job(self, job_id: str):
+        self.scheduler.pause_job(job_id)
+
+    def resume_job(self, job_id: str):
+        self.scheduler.resume_job(job_id)
+
+    def remove_job(self, job_id: str):
+        self.scheduler.remove_job(job_id)
+
+    def execute_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                job.job_func()
+
+    def pause_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.pause_job(job.id)
+
+    def resume_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.resume_job(job.id)
+
+    def remove_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.remove_job(job.id)
+
+    def clear(self):
+        self.__scheduler_is_running = False
+        self.scheduler.remove_all_jobs()
+
+    def shutdown(self):
+        self.scheduler.shutdown()
 
 cron_manager = CronManager()
 
+# FastAPI Routes
 router = APIRouter(prefix="/api/cron", tags=["cron"])
+
 @router.get("/list")
 def _list():
-    def __format(job: schedule.Job) -> dict:
+    def __format(job):
+        return {
+            "id": job.id,
+            "name": job.name,
+            "func": job.func_ref,
+            "pending": job.pending,
+            "trigger": str(job.trigger),
+            "next_run_time": job.next_run_time
+        }
+    return [__format(job) for job in cron_manager.get_jobs()]
+
+@router.get("/default-jobs")
+def _default_jobs():
+    def __format(job):
+        existing_job = cron_manager.scheduler.get_job(job.name)
         return {
-            "job": {'module':job.job_func.__module__,'name':job.job_func.__name__},
-            "at": job.at_time,
-            "interval": job.interval,
-            "last_run": job.last_run,
-            "next_run": job.next_run,
-            "tags": job.tags}
-    _list = cron_manager.get_jobs()
-    return [__format(_) for _ in _list]
+            "name": job.name,
+            "status": "exists" if existing_job else "not added"
+        }
+    return [__format(job) for job in CronManager._list_default]
+
+@router.post("/execute-job/{job_id}")
+def _execute_job(job_id: str):
+    try:
+        cron_manager.execute_job(job_id)
+        return {"status": f"Job {job_id} executed"}
+    except ValueError as e:
+        return {"error": str(e)}
+
+@router.post("/pause-job/{job_id}")
+def _pause_job(job_id: str):
+    cron_manager.pause_job(job_id)
+    return {"status": f"Job {job_id} paused"}
+
+@router.post("/resume-job/{job_id}")
+def _resume_job(job_id: str):
+    cron_manager.resume_job(job_id)
+    return {"status": f"Job {job_id} resumed"}
+
+@router.delete("/remove-job/{job_id}")
+def _remove_job(job_id: str):
+    cron_manager.remove_job(job_id)
+    return {"status": f"Job {job_id} removed"}
+
+@router.post("/execute-recurring")
+def _execute_recurring():
+    cron_manager.execute_recurring_jobs()
+    return {"status": "All recurring jobs executed"}
+
+@router.post("/pause-recurring")
+def _pause_recurring():
+    cron_manager.pause_recurring_jobs()
+    return {"status": "All recurring jobs paused"}
+
+@router.post("/resume-recurring")
+def _resume_recurring():
+    cron_manager.resume_recurring_jobs()
+    return {"status": "All recurring jobs resumed"}
+
+@router.delete("/remove-recurring")
+def _remove_recurring():
+    cron_manager.remove_recurring_jobs()
+    return {"status": "All recurring jobs removed"}
 
 @router.get("/start")
 def _start():
     cron_manager.start()
+    return {"status": "started"}
+
 @router.delete("/stop")
 def _stop():
-    return {"_": cron_manager.clear()}
-@router.get("/run/pending")
-def _run_pending():
-    return {"_": cron_manager.run_pending()}
-@router.get("/run/all")
-def _run_all():
-    return {"_": cron_manager.run_all()}
+    cron_manager.clear()
+    return {"status": "stopped"}
+
+@router.get("/shutdown")
+def _shutdown():
+    cron_manager.shutdown()
+    return {"status": "shutdown"}

ws_bom_robot_app/llm/utils/webhooks.py

@@ -51,3 +51,4 @@ class WebhookNotifier:
         async with httpx.AsyncClient(headers=_headers,verify=False,timeout=timeout) as client:
           response = await client.post(endpoint, data=_data)
           response.raise_for_status()
+

ws_bom_robot_app/llm/vector_store/integration/sitemap.py

@@ -5,7 +5,6 @@ from langchain_community.document_loaders.sitemap import SitemapLoader
 from langchain_community.document_transformers import MarkdownifyTransformer as markdownify
 from langchain_core.documents import Document
 from bs4 import BeautifulSoup, Tag
-import nest_asyncio, os
 
 
 class Sitemap(IntegrationStrategy):

ws_bom_robot_app/main.py

@@ -1,5 +1,6 @@
 import datetime
 import platform
+from fastapi.responses import FileResponse
 import uvicorn, os, sys
 from fastapi import FastAPI, Depends
 from fastapi.openapi.docs import get_swagger_ui_html
@@ -22,7 +23,10 @@ app.include_router(cron,dependencies=[Depends(authenticate)])
 
 @app.get("/")
 async def root():
-    return {}
+    return health()
+@app.get("/favicon.ico")
+async def favicon():
+    return FileResponse("./favicon.ico")
 
 @app.get("/docs", include_in_schema=False)
 async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
@@ -31,7 +35,11 @@ async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
 async def openapi(authenticate: bool = Depends(authenticate)):
     return get_openapi(title=app.title, version=app.version, routes=app.routes)
 
-@app.get("/diag",tags=["diag"])
+@app.get("/api/health",tags=["diag"])
+def health():
+    return {"status": "ok"}
+
+@app.get("/api/diag",tags=["diag"])
 def diag(authenticate: bool = Depends(authenticate)):
     import pkg_resources
     from ws_bom_robot_app.llm.vector_store.loader.base import Loader as wsll
@@ -57,6 +65,7 @@ def diag(authenticate: bool = Depends(authenticate)):
                 "args": {k: arg for k, arg in enumerate(sys.argv)}
             },
             "os": {
+                "ppid": os.getppid(),
                 "pid": os.getpid(),
                 "cwd": os.getcwd(),
                 "ws_bom_robot_app": pkg_resources.get_distribution("ws_bom_robot_app").version,
@@ -64,6 +73,7 @@ def diag(authenticate: bool = Depends(authenticate)):
             },
         },
         "config":config,
+        "runtime":config.runtime_options(),
         "extension": {
             "loader": ({item[0]: item[1].loader.__name__ if item[1] else None} for item in sorted(wsll._list.items(), key=lambda x: x[0]) if item[1]),
             "integration":({item[0]: type(item[1]).__name__} for item in wsim._list.items()),

ws_bom_robot_app/task_manager.py

@@ -2,16 +2,34 @@ import inspect
 import asyncio, os
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Annotated, TypeVar, Optional, Dict, Union, Any
+from typing import Annotated, Literal, TypeVar, Optional, Dict, Union, Any
 from pydantic import BaseModel, ConfigDict, Field, computed_field
 from uuid import uuid4
 from fastapi import APIRouter, HTTPException
 from ws_bom_robot_app.config import config
 from ws_bom_robot_app.llm.models.base import IdentifiableEntity
 from ws_bom_robot_app.llm.utils.webhooks import WebhookNotifier
+from ws_bom_robot_app.util import _log
+from sqlalchemy import create_engine, Column, String, JSON, DateTime, Enum
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from abc import ABC, abstractmethod
 
 T = TypeVar('T')
 
+#region models
+class TaskStatistics(BaseModel):
+    class TaskStatisticExecutionTime(BaseModel):
+        min: str
+        max: str
+        avg: str
+    retention_days: float = config.robot_task_retention_days
+    total: int
+    pending: int
+    completed: int
+    failure: int
+    exec_time: TaskStatisticExecutionTime
+
 class TaskHeader(BaseModel):
     x_ws_bom_msg_type: Optional[str] = None
     x_ws_bom_webhooks: Optional[str] = None
@@ -20,31 +38,24 @@ class TaskHeader(BaseModel):
     )
 
 class TaskMetaData(BaseModel):
-    start_time: datetime
-    end_time: Optional[datetime] = None
+    start_time: str
+    end_time: Optional[str] = None
     @computed_field
     @property
-    def elapsed_time(self) -> Union[timedelta, None]:
-        return (datetime.now() if not self.end_time else self.end_time) - self.start_time
+    def elapsed_time(self) -> Union[str, None]:
+        return str((datetime.now() if not self.end_time else datetime.fromisoformat(self.end_time)) - datetime.fromisoformat(self.start_time))
     source: Optional[str] = None
     pid: Optional[int] = None
-    model_config = ConfigDict(
-        json_encoders={
-            datetime: lambda v: v.isoformat(),
-            timedelta: lambda v: str(v)
-        }
-    )
 
 class TaskStatus(IdentifiableEntity):
-    class TaskStatusEnum(str, Enum):
-        pending = "pending"
-        completed = "completed"
-        failure = "failure"
     type: Optional[str] = None
-    status: TaskStatusEnum
+    status: Literal["pending", "completed", "failure"]
     result: Optional[T] = None
     metadata: TaskMetaData = None
     error: Optional[str] = None
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
 
 class TaskEntry(IdentifiableEntity):
     task: Annotated[asyncio.Task, Field(default=None, validate_default=False)] = None
@@ -65,9 +76,9 @@ class TaskEntry(IdentifiableEntity):
           self.status = TaskStatus(
               id=self.id,
               type=self.headers.x_ws_bom_msg_type if self.headers and self.headers.x_ws_bom_msg_type else self._get_coroutine_name(task._coro) if task else None,
-              status=TaskStatus.TaskStatusEnum.pending,
+              status="pending",
               metadata=TaskMetaData(
-                 start_time=datetime.now(),
+                 start_time=str(datetime.now().isoformat()),
                  source=self._get_coroutine_name(task._coro) if task else None,
                  pid=os.getpid())
               )
@@ -76,76 +87,225 @@ class TaskEntry(IdentifiableEntity):
         validate_assignment=True
     )
 
-class TaskManager:
-    def __init__(self):
-        self.tasks: Dict[str, TaskEntry] = {}
+#endregion
+
+#region interface
+class TaskManagerStrategy(ABC):
+    @abstractmethod
+    def create_task(self, coroutine, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        pass
+
+    @abstractmethod
+    def update_task_status(self, task: TaskEntry) -> None:
+        """Hook for additional behavior, such as persisting the task status."""
+        pass
+
+    @abstractmethod
+    def get_task(self, id: str) -> TaskStatus | None:
+        pass
+
+    @abstractmethod
+    def get_tasks(self) -> list[TaskStatus] | None:
+        pass
+
+    @abstractmethod
+    def remove_task(self, id: str) -> None:
+        pass
 
-    def _task_done_callback(self, task_id: str, headers: TaskHeader | None = None):
+    @abstractmethod
+    def cleanup_task(self) -> None:
+        pass
+
+    @abstractmethod
+    def stats(self) -> TaskStatistics:
+        pass
+
+    def task_cleanup_rule(self, task: TaskEntry) -> bool:
+        return task.status.status in {"completed", "failure"} and datetime.fromisoformat(task.status.metadata.end_time) < datetime.now() - timedelta(days=config.robot_task_retention_days)
+
+    def task_done_callback(self, task_entry: TaskEntry, headers: TaskHeader | None = None) -> callable:
         def callback(task: asyncio.Task):
-            if _task := self.tasks.get(task_id):
-              try:
-                  result = _task.task.result()
-                  _task.status.status = TaskStatus.TaskStatusEnum.completed
-                  _task.status.result = result
-              except Exception as e:
-                  _task.status.status = TaskStatus.TaskStatusEnum.failure
-                  _task.status.error = str(e)
-              finally:
-                  _task.status.metadata.end_time = datetime.now()
-                  if headers and headers.x_ws_bom_webhooks:
-                        asyncio.create_task(
-                            WebhookNotifier().notify_webhook(_task.status,headers.x_ws_bom_webhooks)
-                            )
+            try:
+                result = task_entry.task.result()
+                task_entry.status.status = "completed"
+                task_entry.status.result = result
+            except Exception as e:
+                task_entry.status.status = "failure"
+                task_entry.status.error = str(e)
+            finally:
+                task_entry.status.metadata.end_time = str(datetime.now().isoformat())
+                #strategy-specific behavior
+                self.update_task_status(task_entry)
+                #notify webhooks
+                if headers and headers.x_ws_bom_webhooks:
+                      asyncio.create_task(
+                          WebhookNotifier().notify_webhook(task_entry.status,headers.x_ws_bom_webhooks)
+                          )
         return callback
-
-    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
-        _task = asyncio.create_task(coroutine)
+    def create_task_entry(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> TaskEntry:
         task = TaskEntry(
             id=str(uuid4()),
-            task=_task,
+            task=asyncio.create_task(coroutine),
             headers=headers)
-        task.task.add_done_callback(self._task_done_callback(task.id, headers))
+        task.task.add_done_callback(self.task_done_callback(task, headers))
+        return task
+
+    def stats(self) -> TaskStatistics:
+        def __string_to_timedelta(value: str) -> timedelta:
+            if "." in value:
+                time_format = "%H:%M:%S.%f"
+            else:
+                time_format = "%H:%M:%S"
+            time_obj = datetime.strptime(value, time_format)
+            return timedelta(hours=time_obj.hour, minutes=time_obj.minute, seconds=time_obj.second, microseconds=time_obj.microsecond)
+        def __timedelta_to_string(td):
+            hours, remainder = divmod(td.total_seconds(), 3600)
+            minutes, seconds = divmod(remainder, 60)
+            return f"{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d}.{td.microseconds}"
+        _all = self.get_tasks()
+        _not_pending = _all and [task for task in _all if task.status != "pending"]
+        _total_not_pending = len(_not_pending)
+        elapsed_times = [__string_to_timedelta(task.metadata.elapsed_time) for task in _not_pending]
+        _avg_exec_time = sum(elapsed_times, timedelta()) / _total_not_pending if _total_not_pending > 0 else timedelta()
+        _min_exec_time = min(elapsed_times) if _total_not_pending > 0 else timedelta()
+        _max_exec_time = max(elapsed_times) if _total_not_pending > 0 else timedelta()
+        return TaskStatistics(
+            total= _all and len(_all) or 0,
+            pending=_all and len([task for task in _all if task.status == "pending"]) or 0,
+            completed=_all and len([task for task in _all if task.status == "completed"]) or 0,
+            failure=_all and len([task for task in _all if task.status == "failure"]) or 0,
+            exec_time=TaskStatistics.TaskStatisticExecutionTime(
+                min=__timedelta_to_string(_min_exec_time),
+                max=__timedelta_to_string(_max_exec_time),
+                avg=__timedelta_to_string(_avg_exec_time)
+            )
+        )
+
+#endregion
+
+#memory implementation
+class MemoryTaskManagerStrategy(TaskManagerStrategy):
+    def __init__(self):
+        self.tasks: Dict[str, TaskEntry] = {}
+
+    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        task = self.create_task_entry(coroutine, headers)
         self.tasks[task.id] = task
         return IdentifiableEntity(id=task.id)
 
-    def get_task(self, task_id: str) -> TaskEntry | None:
-        if _task := self.tasks.get(task_id):
+    def update_task_status(self, task: TaskEntry) -> None:
+        """no-op for memory strategy."""
+        pass
+
+    def get_task(self, id: str) -> TaskStatus | None:
+        if _task := self.tasks.get(id):
             return _task
         return None
 
-    def remove_task(self, task_id: str) -> None:
-        if task_id in self.tasks:
-            del self.tasks[task_id]
+    def get_tasks(self) -> list[TaskStatus] | None:
+        return [task.status for task in self.tasks.values()]
+
+    def remove_task(self, id: str) -> None:
+        if id in self.tasks:
+            del self.tasks[id]
 
     def cleanup_task(self):
-        for task_id in [task_id for task_id, task in self.tasks.items()
-                        if task.status.status in {TaskStatus.TaskStatusEnum.completed, TaskStatus.TaskStatusEnum.failure}
-                        and task.status.metadata.end_time < datetime.now() - timedelta(days=config.robot_task_retention_days)]:
-            self.remove_task(task_id)
+        keys = [task.id for task in self.tasks.values() if self.task_cleanup_rule(task)]
+        for key in keys:
+            self.remove_task(key)
+
+#endregion
+
+#db implementation
+Base = declarative_base()
+class TaskEntryModel(Base):
+    __tablename__ = "entry"
+    id = Column(String, primary_key=True)
+    status = Column(JSON)
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
+class DatabaseTaskManagerStrategy(TaskManagerStrategy):
+    def __init__(self, db_url: str = "sqlite:///.data/db/tasks.sqlite"):
+        self.engine = create_engine(db_url)
+        self.Session = sessionmaker(bind=self.engine)
+        Base.metadata.create_all(self.engine)
+
+    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        task = self.create_task_entry(coroutine, headers)
+        with self.Session() as session:
+            session.add(TaskEntryModel(id=task.id, status=task.status.model_dump()))
+            session.commit()
+        return IdentifiableEntity(id=task.id)
+
+    def update_task_status(self, task: TaskEntry) -> None:
+        with self.Session() as session:
+          session.query(TaskEntryModel).filter_by(id=task.id).update(
+              {"status": task.status.model_dump()}
+          )
+          session.commit()
+
+    def get_task(self, id: str) -> TaskStatus | None:
+        with self.Session() as session:
+            task = session.query(TaskEntryModel).filter_by(id=id).first()
+            if task:
+                return TaskEntry(**task.__dict__)
+        return None
+
+    def get_tasks(self) -> list[TaskStatus] | None:
+        with self.Session() as session:
+            tasks = session.query(TaskEntryModel).all()
+            if tasks:
+                return [TaskEntry(**task.__dict__).status for task in tasks]
+        return None
+
+    def remove_task(self, id: str) -> None:
+        with self.Session() as session:
+            session.query(TaskEntryModel).filter_by(id=id).delete()
+            session.commit()
+
+    def cleanup_task(self):
+        with self.Session() as session:
+            for task in session.query(TaskEntryModel).all():
+                _task = TaskEntry(**task.__dict__)
+                if self.task_cleanup_rule(_task):
+                    session.query(TaskEntryModel).filter_by(id=task.id).delete()
+            session.commit()
+#endregion
 
 # global instance
-task_manager = TaskManager()
+def __get_taskmanager_strategy() -> TaskManagerStrategy:
+    if config.runtime_options().is_multi_process:
+        return DatabaseTaskManagerStrategy()
+    return MemoryTaskManagerStrategy()
+task_manager = __get_taskmanager_strategy()
+_log.info(f"Task manager strategy: {task_manager.__class__.__name__}")
 
+#region api
 router = APIRouter(prefix="/api/task", tags=["task"])
-@router.get("/status/{task_id}", response_model=TaskStatus)
-async def _status_task(task_id: str) -> TaskStatus:
-    task = task_manager.get_task(task_id)
+
+@router.get("/status/{id}")
+async def _status_task(id: str) -> TaskStatus:
+    task = task_manager.get_task(id)
     if not task:
         raise HTTPException(status_code=404, detail="Task not found")
     return task.status
+
 @router.get("/status")
-async def _status_task_list():
-    _status_task_list = []
-    for task in task_manager.tasks.values():
-        _task = task_manager.get_task(task.id)
-        _status_task_list.append(_task.status)
-    return _status_task_list
-@router.delete("/status/{task_id}")
-async def _remove_task(task_id: str):
-    task_manager.remove_task(task_id)
-    return {"success":"ok"}
+async def _status_task_list() -> list[TaskStatus]:
+    return task_manager.get_tasks()
+
+@router.delete("/status/{id}")
+async def _remove_task(id: str):
+   task_manager.remove_task(id)
+   return {"success":"ok"}
+
 @router.delete("/cleanup")
 async def _remove_task_list():
     task_manager.cleanup_task()
     return {"success":"ok"}
 
+@router.get("/stats")
+async def _stats() -> TaskStatistics:
+    return task_manager.stats()
+#endregion

{ws_bom_robot_app-0.0.10.dist-info → ws_bom_robot_app-0.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.10
+Version: 0.0.11
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -11,7 +11,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
-Requires-Dist: schedule==1.2.2
+Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: pydantic==2.9.2
 Requires-Dist: pydantic-settings==2.6.0

{ws_bom_robot_app-0.0.10.dist-info → ws_bom_robot_app-0.0.11.dist-info}/RECORD

@@ -1,9 +1,9 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
-ws_bom_robot_app/config.py,sha256=VUoE6MBpYnZ8FQvUjWOLCzfs35aUvpwrz4ROFVbO3AA,1831
-ws_bom_robot_app/cron_manager.py,sha256=Gd76VwhvlauZ5jCrZa4gyM1NoQdynzskfRYPw6niFyk,3548
-ws_bom_robot_app/main.py,sha256=UPRqJkdhFBS8vmJYFJb4hgpx4pMFwOCrbJwt5MTQymE,3556
-ws_bom_robot_app/task_manager.py,sha256=aiQV5EJ_Q0B0mTcERVbzgaEepEUf4OtljC_UACs6pr8,5837
+ws_bom_robot_app/config.py,sha256=V8anZ6Cn8BCv8jO2FMkDxqVA022jnBK2nzOQw6fzf0g,3161
+ws_bom_robot_app/cron_manager.py,sha256=99VZEIEaMmZOYU71bREyBCS2BQVIZG5azTTfOOcxdww,9215
+ws_bom_robot_app/main.py,sha256=JbN4WzbPd0CXW0izVfj4ljxOFeYon-PXfNrhYgbIX14,3869
+ws_bom_robot_app/task_manager.py,sha256=OH-zkWS9iLvCP9eIQdtLsNARMVwIYhBp803ieeNszXo,12012
 ws_bom_robot_app/util.py,sha256=3aBK-bhsvKJwJeWOHh0c1B1BOyJ_tnUxOa1mJmFKwYQ,2618
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_description.py,sha256=SDJYMmwfdMxEK3a_HDEQ19bfNKmwMSFf5hqU0VSCCIE,4705
@@ -29,17 +29,17 @@ ws_bom_robot_app/llm/utils/download.py,sha256=iAUxH_NiCpTPtGzhC4hBtxotd2HPFt2MBh
 ws_bom_robot_app/llm/utils/faiss_helper.py,sha256=DowmroVT6eIbvnA-TG84PS_D7ujvxSRIKdLuIcJmd6Q,4650
 ws_bom_robot_app/llm/utils/kb.py,sha256=jja45WCbNI7SGEgqDS99nErlwB5eY8Ga7BMnhdMHZ90,1279
 ws_bom_robot_app/llm/utils/print.py,sha256=bpLWY0KHXe7x7PWcWy8NS54ZWzHY8b4jrLRkpnDl108,818
-ws_bom_robot_app/llm/utils/webhooks.py,sha256=Sdwu4dkbUC3JEf9ck1hsu4gSGQdj7gcsIrhs3cOVXxk,2112
+ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/generator.py,sha256=DM1jQB6nb03GYFMaO-qN6IX29apFZOYkxEKt6wrXdIQ,5770
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=eCKD3U0KPoVDMtKr2iZqauMFEKd9b2k6rqPG_YjDy0g,626
 ws_bom_robot_app/llm/vector_store/integration/manager.py,sha256=cSFlE2erMv3Uchy788mlCFdcvmyeoqdeIiGmJ9QbLhY,583
-ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=O7E6FfE0gUg6kMkJUy8t19ZnJ74zm_nusNrWWLz_zjo,4031
+ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=A34iaXv80tUgEPf-ntEmC_Pb8GDDAvmfOnloBNOk_Io,4006
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=YKtJWTVQuOvRR2CLYTa0bGEbjqgmYVVud1YKQ9QLibc,4536
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=G9BoxwsevgqL72h2n28O2LpzaCYNymBkX66wru9GkCw,884
-ws_bom_robot_app-0.0.10.dist-info/METADATA,sha256=OfGHsmNN39wqXEptnM_p1en48OhEKQHN0KvCN1SsBp4,5674
-ws_bom_robot_app-0.0.10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ws_bom_robot_app-0.0.10.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.10.dist-info/RECORD,,
+ws_bom_robot_app-0.0.11.dist-info/METADATA,sha256=SA7_BZPmnAQ5-v2oXHWJgzF7zLc_js09h6pl96wHFzU,5678
+ws_bom_robot_app-0.0.11.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ws_bom_robot_app-0.0.11.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.11.dist-info/RECORD,,