ws-bom-robot-app 0.0.10__tar.gz → 0.0.11__tar.gz

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.10
+Version: 0.0.11
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -11,7 +11,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
-Requires-Dist: schedule==1.2.2
+Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: pydantic==2.9.2
 Requires-Dist: pydantic-settings==2.6.0

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.10",
+    version="0.0.11",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app/config.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from pydantic import BaseModel, ConfigDict
 from pydantic_settings import BaseSettings
 
@@ -25,10 +26,38 @@ class Settings(BaseSettings):
     )
 
     class RuntimeOptions(BaseModel):
+        def _is_multi_process() -> bool:
+            """
+            Checks if the application is running with multiple worker processes.
+
+            This function inspects the command-line arguments to determine if the
+            application is configured to run with more than one worker process. It
+            looks for the "--workers" argument and checks if the subsequent value
+            is greater than 1.
+            Sample of command-line arguments:
+            fastapi run main.py --port 6001 --workers 4
+            uvicorn main:app --port 6001 --workers 4
+
+            Returns:
+                bool: True if the application is running with multiple worker
+                      processes, False otherwise.
+            """
+            import sys, os
+            try:
+                for i, arg in enumerate(sys.argv):
+                    if arg == "--workers" and i + 1 < len(sys.argv):
+                        workers = int(sys.argv[i + 1])
+                        if workers > 1:
+                            return True
+            except (ValueError, IndexError):
+                pass
+            # Fallback: Compare process and parent process IDs
+            return False #os.getpid() != os.getppid()
         debug: bool
         loader_strategy: str
         loader_show_progress: bool
         loader_silent_errors: bool
+        is_multi_process: bool = _is_multi_process()
 
 
     def runtime_options(self) -> RuntimeOptions:

ws_bom_robot_app-0.0.11/ws_bom_robot_app/cron_manager.py

@@ -0,0 +1,250 @@
+from apscheduler.schedulers.background import BackgroundScheduler
+#from apscheduler.schedulers.asyncio import AsyncIOScheduler
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
+from apscheduler.triggers.cron import CronTrigger
+from apscheduler.triggers.interval import IntervalTrigger
+from apscheduler.triggers.date import DateTrigger
+from fastapi import APIRouter
+from datetime import datetime
+from ws_bom_robot_app.task_manager import task_manager
+from ws_bom_robot_app.llm.utils.kb import kb_cleanup_data_file
+from ws_bom_robot_app.util import _log
+from ws_bom_robot_app.config import config
+
+class JobstoreStrategy:
+    def get_jobstore(self):
+        raise NotImplementedError("Subclasses should implement this method")
+
+class MemoryJobstoreStrategy(JobstoreStrategy):
+    def get_jobstore(self):
+        _log.info("Using in-memory cron jobstore.")
+        return {"default": MemoryJobStore()}
+
+class PersistentJobstoreStrategy(JobstoreStrategy):
+    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
+        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+        return {"default": SQLAlchemyJobStore(url=db_url)}
+
+class Job:
+    def __init__(self, name: str, job_func, args: list = None, kwargs: dict = None, cron_expression: str = None, interval: int = None, run_at: datetime = None):
+        """
+        Job class that supports both recurring and one-time jobs.
+        :param job_func: The function to execute.
+        :param interval: Interval in seconds for recurring jobs.
+        :param run_at: Specific datetime for one-time jobs.
+        :param tags: Tags associated with the job.
+        """
+        if not (cron_expression or interval or run_at):
+            raise ValueError("Either 'interval' or 'run_at' must be provided.")
+        self.name = name
+        self.job_func = job_func
+        self.args: list = args or []
+        self.kwargs: dict = kwargs or {}
+        self.cron_expression = cron_expression
+        self.interval = interval
+        self.run_at = run_at
+
+    def create_trigger(self):
+        """Create the appropriate trigger based on the job type."""
+        if self.cron_expression:
+            return CronTrigger.from_crontab(self.cron_expression)
+        if self.interval:
+            return IntervalTrigger(seconds=self.interval)
+        elif self.run_at:
+            return DateTrigger(run_date=self.run_at)
+
+class CronManager:
+    _list_default = [
+            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
+            Job('cleanup-data',kb_cleanup_data_file, interval=180 * 60),
+        ]
+    def __get_jobstore_strategy() -> JobstoreStrategy:
+        if config.runtime_options().is_multi_process:
+            return MemoryJobstoreStrategy()
+        return PersistentJobstoreStrategy()
+    def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):
+        self.enable_defaults = enable_defaults
+        if strategy is None:
+          strategy = CronManager.__get_jobstore_strategy()
+        jobstores = strategy.get_jobstore()
+        self.scheduler: BackgroundScheduler = BackgroundScheduler(jobstores=jobstores)
+        self.__scheduler_is_running = False
+
+    def add_job(self, job: Job):
+        """
+        Adds a job to the scheduler with the specified name and job details.
+        Args:
+          name (str): The unique identifier for the job.
+          job (Job): An instance of the Job class containing the job details.
+        The job details include:
+          - job_func: The function to be executed.
+          - args: The positional arguments to pass to the job function.
+          - kwargs: The keyword arguments to pass to the job function.
+          - trigger: The trigger that determines when the job should be executed.
+        The job will replace any existing job with the same name.
+        Sample usage:
+          recurring_job = Job(name="sample-recurring-job",job_func=example_job, interval=5, tags=tags, args=args, kwargs=kwargs)
+          cron_manager.add_job(recurring_job)
+          fire_once_job = Job(name="sample-fire-once-job",job_func=example_job, run_at=datetime.now(), tags=tags, args=args, kwargs=kwargs)
+          cron_manager.add_job(fire_once_job)
+        """
+        existing_job = self.scheduler.get_job(job.name)
+        if existing_job:
+            _log.info(f"Job with name '{job.name}' already exists. Skip creation.")
+        else:
+          trigger = job.create_trigger()
+          self.scheduler.add_job(
+              func=job.job_func,
+              args=job.args,
+              kwargs=job.kwargs,
+              trigger=trigger,
+              id=job.name,
+              name=job.name,
+              replace_existing=False
+          )
+
+    def start(self):
+        if not self.__scheduler_is_running:
+            self.__scheduler_is_running = True
+            self.scheduler.start()
+            if self.enable_defaults and CronManager._list_default:
+                for job in CronManager._list_default:
+                    existing_job = self.scheduler.get_job(job.name)
+                    if existing_job is None:
+                        self.add_job(job)
+
+    def get_job(self, job_id: str):
+        return self.scheduler.get_job(job_id)
+
+    def get_jobs(self):
+        return self.scheduler.get_jobs()
+
+    def execute_job(self, job_id: str):
+        job = self.scheduler.get_job(job_id)
+        if job:
+            job.func()
+        else:
+            raise ValueError(f"Job with id '{job_id}' not found.")
+
+    def pause_job(self, job_id: str):
+        self.scheduler.pause_job(job_id)
+
+    def resume_job(self, job_id: str):
+        self.scheduler.resume_job(job_id)
+
+    def remove_job(self, job_id: str):
+        self.scheduler.remove_job(job_id)
+
+    def execute_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                job.job_func()
+
+    def pause_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.pause_job(job.id)
+
+    def resume_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.resume_job(job.id)
+
+    def remove_recurring_jobs(self):
+        for job in self.scheduler.get_jobs():
+            if job.interval:
+                self.remove_job(job.id)
+
+    def clear(self):
+        self.__scheduler_is_running = False
+        self.scheduler.remove_all_jobs()
+
+    def shutdown(self):
+        self.scheduler.shutdown()
+
+cron_manager = CronManager()
+
+# FastAPI Routes
+router = APIRouter(prefix="/api/cron", tags=["cron"])
+
+@router.get("/list")
+def _list():
+    def __format(job):
+        return {
+            "id": job.id,
+            "name": job.name,
+            "func": job.func_ref,
+            "pending": job.pending,
+            "trigger": str(job.trigger),
+            "next_run_time": job.next_run_time
+        }
+    return [__format(job) for job in cron_manager.get_jobs()]
+
+@router.get("/default-jobs")
+def _default_jobs():
+    def __format(job):
+        existing_job = cron_manager.scheduler.get_job(job.name)
+        return {
+            "name": job.name,
+            "status": "exists" if existing_job else "not added"
+        }
+    return [__format(job) for job in CronManager._list_default]
+
+@router.post("/execute-job/{job_id}")
+def _execute_job(job_id: str):
+    try:
+        cron_manager.execute_job(job_id)
+        return {"status": f"Job {job_id} executed"}
+    except ValueError as e:
+        return {"error": str(e)}
+
+@router.post("/pause-job/{job_id}")
+def _pause_job(job_id: str):
+    cron_manager.pause_job(job_id)
+    return {"status": f"Job {job_id} paused"}
+
+@router.post("/resume-job/{job_id}")
+def _resume_job(job_id: str):
+    cron_manager.resume_job(job_id)
+    return {"status": f"Job {job_id} resumed"}
+
+@router.delete("/remove-job/{job_id}")
+def _remove_job(job_id: str):
+    cron_manager.remove_job(job_id)
+    return {"status": f"Job {job_id} removed"}
+
+@router.post("/execute-recurring")
+def _execute_recurring():
+    cron_manager.execute_recurring_jobs()
+    return {"status": "All recurring jobs executed"}
+
+@router.post("/pause-recurring")
+def _pause_recurring():
+    cron_manager.pause_recurring_jobs()
+    return {"status": "All recurring jobs paused"}
+
+@router.post("/resume-recurring")
+def _resume_recurring():
+    cron_manager.resume_recurring_jobs()
+    return {"status": "All recurring jobs resumed"}
+
+@router.delete("/remove-recurring")
+def _remove_recurring():
+    cron_manager.remove_recurring_jobs()
+    return {"status": "All recurring jobs removed"}
+
+@router.get("/start")
+def _start():
+    cron_manager.start()
+    return {"status": "started"}
+
+@router.delete("/stop")
+def _stop():
+    cron_manager.clear()
+    return {"status": "stopped"}
+
+@router.get("/shutdown")
+def _shutdown():
+    cron_manager.shutdown()
+    return {"status": "shutdown"}

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app/llm/utils/webhooks.py

@@ -51,3 +51,4 @@ class WebhookNotifier:
         async with httpx.AsyncClient(headers=_headers,verify=False,timeout=timeout) as client:
           response = await client.post(endpoint, data=_data)
           response.raise_for_status()
+

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app/llm/vector_store/integration/sitemap.py

@@ -5,7 +5,6 @@ from langchain_community.document_loaders.sitemap import SitemapLoader
 from langchain_community.document_transformers import MarkdownifyTransformer as markdownify
 from langchain_core.documents import Document
 from bs4 import BeautifulSoup, Tag
-import nest_asyncio, os
 
 
 class Sitemap(IntegrationStrategy):

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app/main.py

@@ -1,5 +1,6 @@
 import datetime
 import platform
+from fastapi.responses import FileResponse
 import uvicorn, os, sys
 from fastapi import FastAPI, Depends
 from fastapi.openapi.docs import get_swagger_ui_html
@@ -22,7 +23,10 @@ app.include_router(cron,dependencies=[Depends(authenticate)])
 
 @app.get("/")
 async def root():
-    return {}
+    return health()
+@app.get("/favicon.ico")
+async def favicon():
+    return FileResponse("./favicon.ico")
 
 @app.get("/docs", include_in_schema=False)
 async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
@@ -31,7 +35,11 @@ async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
 async def openapi(authenticate: bool = Depends(authenticate)):
     return get_openapi(title=app.title, version=app.version, routes=app.routes)
 
-@app.get("/diag",tags=["diag"])
+@app.get("/api/health",tags=["diag"])
+def health():
+    return {"status": "ok"}
+
+@app.get("/api/diag",tags=["diag"])
 def diag(authenticate: bool = Depends(authenticate)):
     import pkg_resources
     from ws_bom_robot_app.llm.vector_store.loader.base import Loader as wsll
@@ -57,6 +65,7 @@ def diag(authenticate: bool = Depends(authenticate)):
                 "args": {k: arg for k, arg in enumerate(sys.argv)}
             },
             "os": {
+                "ppid": os.getppid(),
                 "pid": os.getpid(),
                 "cwd": os.getcwd(),
                 "ws_bom_robot_app": pkg_resources.get_distribution("ws_bom_robot_app").version,
@@ -64,6 +73,7 @@ def diag(authenticate: bool = Depends(authenticate)):
             },
         },
         "config":config,
+        "runtime":config.runtime_options(),
         "extension": {
             "loader": ({item[0]: item[1].loader.__name__ if item[1] else None} for item in sorted(wsll._list.items(), key=lambda x: x[0]) if item[1]),
             "integration":({item[0]: type(item[1]).__name__} for item in wsim._list.items()),

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app/requirements.txt

@@ -1,6 +1,6 @@
 #app
 standardwebhooks==1.0.0
-schedule==1.2.2
+apscheduler==3.11.0
 aiofiles==24.1.0
 pydantic==2.9.2
 pydantic-settings==2.6.0

ws_bom_robot_app-0.0.11/ws_bom_robot_app/task_manager.py

@@ -0,0 +1,311 @@
+import inspect
+import asyncio, os
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import Annotated, Literal, TypeVar, Optional, Dict, Union, Any
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+from uuid import uuid4
+from fastapi import APIRouter, HTTPException
+from ws_bom_robot_app.config import config
+from ws_bom_robot_app.llm.models.base import IdentifiableEntity
+from ws_bom_robot_app.llm.utils.webhooks import WebhookNotifier
+from ws_bom_robot_app.util import _log
+from sqlalchemy import create_engine, Column, String, JSON, DateTime, Enum
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from abc import ABC, abstractmethod
+
+T = TypeVar('T')
+
+#region models
+class TaskStatistics(BaseModel):
+    class TaskStatisticExecutionTime(BaseModel):
+        min: str
+        max: str
+        avg: str
+    retention_days: float = config.robot_task_retention_days
+    total: int
+    pending: int
+    completed: int
+    failure: int
+    exec_time: TaskStatisticExecutionTime
+
+class TaskHeader(BaseModel):
+    x_ws_bom_msg_type: Optional[str] = None
+    x_ws_bom_webhooks: Optional[str] = None
+    model_config = ConfigDict(
+        extra='allow'
+    )
+
+class TaskMetaData(BaseModel):
+    start_time: str
+    end_time: Optional[str] = None
+    @computed_field
+    @property
+    def elapsed_time(self) -> Union[str, None]:
+        return str((datetime.now() if not self.end_time else datetime.fromisoformat(self.end_time)) - datetime.fromisoformat(self.start_time))
+    source: Optional[str] = None
+    pid: Optional[int] = None
+
+class TaskStatus(IdentifiableEntity):
+    type: Optional[str] = None
+    status: Literal["pending", "completed", "failure"]
+    result: Optional[T] = None
+    metadata: TaskMetaData = None
+    error: Optional[str] = None
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
+
+class TaskEntry(IdentifiableEntity):
+    task: Annotated[asyncio.Task, Field(default=None, validate_default=False)] = None
+    headers: TaskHeader | None = None
+    status: Union[TaskStatus, None] = None
+    def _get_coroutine_name(self, coroutine: asyncio.coroutines) -> str:
+        if inspect.iscoroutine(coroutine):
+            return coroutine.cr_code.co_name
+        return "<unknown>"
+    def __init__(self, **data):
+        #separate task from data to handle asyncio.Task
+        task = data.pop('task',None)
+        super().__init__(**data)
+        #bypass pydantic validation
+        object.__setattr__(self, 'task', task)
+        #init status
+        if not self.status:
+          self.status = TaskStatus(
+              id=self.id,
+              type=self.headers.x_ws_bom_msg_type if self.headers and self.headers.x_ws_bom_msg_type else self._get_coroutine_name(task._coro) if task else None,
+              status="pending",
+              metadata=TaskMetaData(
+                 start_time=str(datetime.now().isoformat()),
+                 source=self._get_coroutine_name(task._coro) if task else None,
+                 pid=os.getpid())
+              )
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        validate_assignment=True
+    )
+
+#endregion
+
+#region interface
+class TaskManagerStrategy(ABC):
+    @abstractmethod
+    def create_task(self, coroutine, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        pass
+
+    @abstractmethod
+    def update_task_status(self, task: TaskEntry) -> None:
+        """Hook for additional behavior, such as persisting the task status."""
+        pass
+
+    @abstractmethod
+    def get_task(self, id: str) -> TaskStatus | None:
+        pass
+
+    @abstractmethod
+    def get_tasks(self) -> list[TaskStatus] | None:
+        pass
+
+    @abstractmethod
+    def remove_task(self, id: str) -> None:
+        pass
+
+    @abstractmethod
+    def cleanup_task(self) -> None:
+        pass
+
+    @abstractmethod
+    def stats(self) -> TaskStatistics:
+        pass
+
+    def task_cleanup_rule(self, task: TaskEntry) -> bool:
+        return task.status.status in {"completed", "failure"} and datetime.fromisoformat(task.status.metadata.end_time) < datetime.now() - timedelta(days=config.robot_task_retention_days)
+
+    def task_done_callback(self, task_entry: TaskEntry, headers: TaskHeader | None = None) -> callable:
+        def callback(task: asyncio.Task):
+            try:
+                result = task_entry.task.result()
+                task_entry.status.status = "completed"
+                task_entry.status.result = result
+            except Exception as e:
+                task_entry.status.status = "failure"
+                task_entry.status.error = str(e)
+            finally:
+                task_entry.status.metadata.end_time = str(datetime.now().isoformat())
+                #strategy-specific behavior
+                self.update_task_status(task_entry)
+                #notify webhooks
+                if headers and headers.x_ws_bom_webhooks:
+                      asyncio.create_task(
+                          WebhookNotifier().notify_webhook(task_entry.status,headers.x_ws_bom_webhooks)
+                          )
+        return callback
+    def create_task_entry(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> TaskEntry:
+        task = TaskEntry(
+            id=str(uuid4()),
+            task=asyncio.create_task(coroutine),
+            headers=headers)
+        task.task.add_done_callback(self.task_done_callback(task, headers))
+        return task
+
+    def stats(self) -> TaskStatistics:
+        def __string_to_timedelta(value: str) -> timedelta:
+            if "." in value:
+                time_format = "%H:%M:%S.%f"
+            else:
+                time_format = "%H:%M:%S"
+            time_obj = datetime.strptime(value, time_format)
+            return timedelta(hours=time_obj.hour, minutes=time_obj.minute, seconds=time_obj.second, microseconds=time_obj.microsecond)
+        def __timedelta_to_string(td):
+            hours, remainder = divmod(td.total_seconds(), 3600)
+            minutes, seconds = divmod(remainder, 60)
+            return f"{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d}.{td.microseconds}"
+        _all = self.get_tasks()
+        _not_pending = _all and [task for task in _all if task.status != "pending"]
+        _total_not_pending = len(_not_pending)
+        elapsed_times = [__string_to_timedelta(task.metadata.elapsed_time) for task in _not_pending]
+        _avg_exec_time = sum(elapsed_times, timedelta()) / _total_not_pending if _total_not_pending > 0 else timedelta()
+        _min_exec_time = min(elapsed_times) if _total_not_pending > 0 else timedelta()
+        _max_exec_time = max(elapsed_times) if _total_not_pending > 0 else timedelta()
+        return TaskStatistics(
+            total= _all and len(_all) or 0,
+            pending=_all and len([task for task in _all if task.status == "pending"]) or 0,
+            completed=_all and len([task for task in _all if task.status == "completed"]) or 0,
+            failure=_all and len([task for task in _all if task.status == "failure"]) or 0,
+            exec_time=TaskStatistics.TaskStatisticExecutionTime(
+                min=__timedelta_to_string(_min_exec_time),
+                max=__timedelta_to_string(_max_exec_time),
+                avg=__timedelta_to_string(_avg_exec_time)
+            )
+        )
+
+#endregion
+
+#memory implementation
+class MemoryTaskManagerStrategy(TaskManagerStrategy):
+    def __init__(self):
+        self.tasks: Dict[str, TaskEntry] = {}
+
+    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        task = self.create_task_entry(coroutine, headers)
+        self.tasks[task.id] = task
+        return IdentifiableEntity(id=task.id)
+
+    def update_task_status(self, task: TaskEntry) -> None:
+        """no-op for memory strategy."""
+        pass
+
+    def get_task(self, id: str) -> TaskStatus | None:
+        if _task := self.tasks.get(id):
+            return _task
+        return None
+
+    def get_tasks(self) -> list[TaskStatus] | None:
+        return [task.status for task in self.tasks.values()]
+
+    def remove_task(self, id: str) -> None:
+        if id in self.tasks:
+            del self.tasks[id]
+
+    def cleanup_task(self):
+        keys = [task.id for task in self.tasks.values() if self.task_cleanup_rule(task)]
+        for key in keys:
+            self.remove_task(key)
+
+#endregion
+
+#db implementation
+Base = declarative_base()
+class TaskEntryModel(Base):
+    __tablename__ = "entry"
+    id = Column(String, primary_key=True)
+    status = Column(JSON)
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
+class DatabaseTaskManagerStrategy(TaskManagerStrategy):
+    def __init__(self, db_url: str = "sqlite:///.data/db/tasks.sqlite"):
+        self.engine = create_engine(db_url)
+        self.Session = sessionmaker(bind=self.engine)
+        Base.metadata.create_all(self.engine)
+
+    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
+        task = self.create_task_entry(coroutine, headers)
+        with self.Session() as session:
+            session.add(TaskEntryModel(id=task.id, status=task.status.model_dump()))
+            session.commit()
+        return IdentifiableEntity(id=task.id)
+
+    def update_task_status(self, task: TaskEntry) -> None:
+        with self.Session() as session:
+          session.query(TaskEntryModel).filter_by(id=task.id).update(
+              {"status": task.status.model_dump()}
+          )
+          session.commit()
+
+    def get_task(self, id: str) -> TaskStatus | None:
+        with self.Session() as session:
+            task = session.query(TaskEntryModel).filter_by(id=id).first()
+            if task:
+                return TaskEntry(**task.__dict__)
+        return None
+
+    def get_tasks(self) -> list[TaskStatus] | None:
+        with self.Session() as session:
+            tasks = session.query(TaskEntryModel).all()
+            if tasks:
+                return [TaskEntry(**task.__dict__).status for task in tasks]
+        return None
+
+    def remove_task(self, id: str) -> None:
+        with self.Session() as session:
+            session.query(TaskEntryModel).filter_by(id=id).delete()
+            session.commit()
+
+    def cleanup_task(self):
+        with self.Session() as session:
+            for task in session.query(TaskEntryModel).all():
+                _task = TaskEntry(**task.__dict__)
+                if self.task_cleanup_rule(_task):
+                    session.query(TaskEntryModel).filter_by(id=task.id).delete()
+            session.commit()
+#endregion
+
+# global instance
+def __get_taskmanager_strategy() -> TaskManagerStrategy:
+    if config.runtime_options().is_multi_process:
+        return DatabaseTaskManagerStrategy()
+    return MemoryTaskManagerStrategy()
+task_manager = __get_taskmanager_strategy()
+_log.info(f"Task manager strategy: {task_manager.__class__.__name__}")
+
+#region api
+router = APIRouter(prefix="/api/task", tags=["task"])
+
+@router.get("/status/{id}")
+async def _status_task(id: str) -> TaskStatus:
+    task = task_manager.get_task(id)
+    if not task:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return task.status
+
+@router.get("/status")
+async def _status_task_list() -> list[TaskStatus]:
+    return task_manager.get_tasks()
+
+@router.delete("/status/{id}")
+async def _remove_task(id: str):
+   task_manager.remove_task(id)
+   return {"success":"ok"}
+
+@router.delete("/cleanup")
+async def _remove_task_list():
+    task_manager.cleanup_task()
+    return {"success":"ok"}
+
+@router.get("/stats")
+async def _stats() -> TaskStatistics:
+    return task_manager.stats()
+#endregion

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ws_bom_robot_app
-Version: 0.0.10
+Version: 0.0.11
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -11,7 +11,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 Requires-Dist: standardwebhooks==1.0.0
-Requires-Dist: schedule==1.2.2
+Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: pydantic==2.9.2
 Requires-Dist: pydantic-settings==2.6.0

{ws_bom_robot_app-0.0.10 → ws_bom_robot_app-0.0.11}/ws_bom_robot_app.egg-info/requires.txt

@@ -1,5 +1,5 @@
 standardwebhooks==1.0.0
-schedule==1.2.2
+apscheduler==3.11.0
 aiofiles==24.1.0
 pydantic==2.9.2
 pydantic-settings==2.6.0

ws_bom_robot_app-0.0.10/ws_bom_robot_app/cron_manager.py

@@ -1,99 +0,0 @@
-from math import floor
-import schedule, time, threading
-import asyncio
-from fastapi import APIRouter
-from ws_bom_robot_app.task_manager import task_manager
-from ws_bom_robot_app.llm.utils.kb import kb_cleanup_data_file
-from ws_bom_robot_app.util import _log
-import random
-
-class RecurringJob():
-    @staticmethod
-    def __add_jitter(interval: int) -> int:
-      #delay proportional with interval, min 10, max 100 sec
-      jitter: int = max(10, min(100, floor(interval * 0.075)))
-      return interval + random.randint(-jitter, jitter)
-    def __init__(self, interval: int, job_func, tags: list[str]):
-        #add a little jitter by default for better concurrency in case of multiple instances
-        self.interval = RecurringJob.__add_jitter(interval)
-        self.job_func = job_func
-        self.is_coroutine = asyncio.iscoroutinefunction(job_func)
-        self.job_func = job_func
-        self.tags = tags
-    def run(self):
-        if self.is_coroutine:
-            schedule.every(self.interval).seconds.do(self._run_async_job).tag(*self.tags)
-        else:
-            schedule.every(self.interval).seconds.do(self.job_func).tag(*self.tags)
-    async def _run_async_job(self):
-        await self.job_func()
-
-class CronManager:
-
-    _list: dict[str, RecurringJob] = {
-      'cleanup-task': RecurringJob(5*60, task_manager.cleanup_task, tags=["cleanup","cleanup-task"]),
-      'cleanup-data': RecurringJob(180*60, kb_cleanup_data_file, tags=["cleanup","cleanup-data"]),
-    }
-
-    def __init__(self):
-        self.jobs: dict[str, RecurringJob] = CronManager._list
-        self.__scheduler_is_running = False
-    def add_job(self, name:str, job: RecurringJob):
-        job = {name: job}
-        self.jobs.append(job)
-        return job
-    def run_pending(self):
-        return schedule.run_pending()
-    def run_all(self):
-        return schedule.run_all()
-    def clear(self):
-        self.__scheduler_is_running = False
-        return schedule.clear()
-    def get_jobs(self):
-        return schedule.get_jobs()
-    def start(self):
-        def _target():
-            while self.__scheduler_is_running:
-                time.sleep(1)
-                self.run_pending()
-                time.sleep(59)
-            _log.info(f"__scheduler_is_running={self.__scheduler_is_running}")
-        #clear all jobs
-        self.clear()
-        #prepare jobs
-        for job in self.jobs.values():
-            job.run()
-        #start scheduler
-        if not self.__scheduler_is_running:
-            self.__scheduler_is_running = True
-            t = threading.Thread(target=_target,args=(),daemon=True)
-            t.start()
-
-cron_manager = CronManager()
-
-router = APIRouter(prefix="/api/cron", tags=["cron"])
-@router.get("/list")
-def _list():
-    def __format(job: schedule.Job) -> dict:
-        return {
-            "job": {'module':job.job_func.__module__,'name':job.job_func.__name__},
-            "at": job.at_time,
-            "interval": job.interval,
-            "last_run": job.last_run,
-            "next_run": job.next_run,
-            "tags": job.tags}
-    _list = cron_manager.get_jobs()
-    return [__format(_) for _ in _list]
-
-@router.get("/start")
-def _start():
-    cron_manager.start()
-@router.delete("/stop")
-def _stop():
-    return {"_": cron_manager.clear()}
-@router.get("/run/pending")
-def _run_pending():
-    return {"_": cron_manager.run_pending()}
-@router.get("/run/all")
-def _run_all():
-    return {"_": cron_manager.run_all()}

ws_bom_robot_app-0.0.10/ws_bom_robot_app/task_manager.py

@@ -1,151 +0,0 @@
-import inspect
-import asyncio, os
-from datetime import datetime, timedelta
-from enum import Enum
-from typing import Annotated, TypeVar, Optional, Dict, Union, Any
-from pydantic import BaseModel, ConfigDict, Field, computed_field
-from uuid import uuid4
-from fastapi import APIRouter, HTTPException
-from ws_bom_robot_app.config import config
-from ws_bom_robot_app.llm.models.base import IdentifiableEntity
-from ws_bom_robot_app.llm.utils.webhooks import WebhookNotifier
-
-T = TypeVar('T')
-
-class TaskHeader(BaseModel):
-    x_ws_bom_msg_type: Optional[str] = None
-    x_ws_bom_webhooks: Optional[str] = None
-    model_config = ConfigDict(
-        extra='allow'
-    )
-
-class TaskMetaData(BaseModel):
-    start_time: datetime
-    end_time: Optional[datetime] = None
-    @computed_field
-    @property
-    def elapsed_time(self) -> Union[timedelta, None]:
-        return (datetime.now() if not self.end_time else self.end_time) - self.start_time
-    source: Optional[str] = None
-    pid: Optional[int] = None
-    model_config = ConfigDict(
-        json_encoders={
-            datetime: lambda v: v.isoformat(),
-            timedelta: lambda v: str(v)
-        }
-    )
-
-class TaskStatus(IdentifiableEntity):
-    class TaskStatusEnum(str, Enum):
-        pending = "pending"
-        completed = "completed"
-        failure = "failure"
-    type: Optional[str] = None
-    status: TaskStatusEnum
-    result: Optional[T] = None
-    metadata: TaskMetaData = None
-    error: Optional[str] = None
-
-class TaskEntry(IdentifiableEntity):
-    task: Annotated[asyncio.Task, Field(default=None, validate_default=False)] = None
-    headers: TaskHeader | None = None
-    status: Union[TaskStatus, None] = None
-    def _get_coroutine_name(self, coroutine: asyncio.coroutines) -> str:
-        if inspect.iscoroutine(coroutine):
-            return coroutine.cr_code.co_name
-        return "<unknown>"
-    def __init__(self, **data):
-        #separate task from data to handle asyncio.Task
-        task = data.pop('task',None)
-        super().__init__(**data)
-        #bypass pydantic validation
-        object.__setattr__(self, 'task', task)
-        #init status
-        if not self.status:
-          self.status = TaskStatus(
-              id=self.id,
-              type=self.headers.x_ws_bom_msg_type if self.headers and self.headers.x_ws_bom_msg_type else self._get_coroutine_name(task._coro) if task else None,
-              status=TaskStatus.TaskStatusEnum.pending,
-              metadata=TaskMetaData(
-                 start_time=datetime.now(),
-                 source=self._get_coroutine_name(task._coro) if task else None,
-                 pid=os.getpid())
-              )
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True,
-        validate_assignment=True
-    )
-
-class TaskManager:
-    def __init__(self):
-        self.tasks: Dict[str, TaskEntry] = {}
-
-    def _task_done_callback(self, task_id: str, headers: TaskHeader | None = None):
-        def callback(task: asyncio.Task):
-            if _task := self.tasks.get(task_id):
-              try:
-                  result = _task.task.result()
-                  _task.status.status = TaskStatus.TaskStatusEnum.completed
-                  _task.status.result = result
-              except Exception as e:
-                  _task.status.status = TaskStatus.TaskStatusEnum.failure
-                  _task.status.error = str(e)
-              finally:
-                  _task.status.metadata.end_time = datetime.now()
-                  if headers and headers.x_ws_bom_webhooks:
-                        asyncio.create_task(
-                            WebhookNotifier().notify_webhook(_task.status,headers.x_ws_bom_webhooks)
-                            )
-        return callback
-
-    def create_task(self, coroutine: asyncio.coroutines, headers: TaskHeader | None = None) -> IdentifiableEntity:
-        _task = asyncio.create_task(coroutine)
-        task = TaskEntry(
-            id=str(uuid4()),
-            task=_task,
-            headers=headers)
-        task.task.add_done_callback(self._task_done_callback(task.id, headers))
-        self.tasks[task.id] = task
-        return IdentifiableEntity(id=task.id)
-
-    def get_task(self, task_id: str) -> TaskEntry | None:
-        if _task := self.tasks.get(task_id):
-            return _task
-        return None
-
-    def remove_task(self, task_id: str) -> None:
-        if task_id in self.tasks:
-            del self.tasks[task_id]
-
-    def cleanup_task(self):
-        for task_id in [task_id for task_id, task in self.tasks.items()
-                        if task.status.status in {TaskStatus.TaskStatusEnum.completed, TaskStatus.TaskStatusEnum.failure}
-                        and task.status.metadata.end_time < datetime.now() - timedelta(days=config.robot_task_retention_days)]:
-            self.remove_task(task_id)
-
-# global instance
-task_manager = TaskManager()
-
-router = APIRouter(prefix="/api/task", tags=["task"])
-@router.get("/status/{task_id}", response_model=TaskStatus)
-async def _status_task(task_id: str) -> TaskStatus:
-    task = task_manager.get_task(task_id)
-    if not task:
-        raise HTTPException(status_code=404, detail="Task not found")
-    return task.status
-@router.get("/status")
-async def _status_task_list():
-    _status_task_list = []
-    for task in task_manager.tasks.values():
-        _task = task_manager.get_task(task.id)
-        _status_task_list.append(_task.status)
-    return _status_task_list
-@router.delete("/status/{task_id}")
-async def _remove_task(task_id: str):
-    task_manager.remove_task(task_id)
-    return {"success":"ok"}
-@router.delete("/cleanup")
-async def _remove_task_list():
-    task_manager.cleanup_task()
-    return {"success":"ok"}
-