ws-bom-robot-app 0.0.84__py3-none-any.whl → 0.0.85__py3-none-any.whl

ws_bom_robot_app/cron_manager.py

@@ -56,9 +56,9 @@ class Job:
 
 class CronManager:
     _list_default = [
-            Job('cleanup-task-history',task_cleanup_history, interval=5 * 60),
-            Job('cleanup-kb-data',kb_cleanup_data_file, interval=180 * 60),
-            Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=120 * 60),
+            Job('cleanup-task-history',task_cleanup_history, interval=4 * 60 * 60),
+            Job('cleanup-kb-data',kb_cleanup_data_file, interval=8 * 60 * 60),
+            Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=6 * 60 * 60),
         ]
     def __get_jobstore_strategy(self) -> JobstoreStrategy:
         if config.robot_cron_strategy == 'memory':

ws_bom_robot_app/llm/utils/cleanup.py

@@ -1,7 +1,6 @@
 import os, logging
 from ws_bom_robot_app.config import config
 from datetime import datetime, timedelta
-from ws_bom_robot_app.task_manager import task_manager
 
 def _cleanup_data_file(folders: list[str], retention: float) -> dict:
     """
@@ -78,4 +77,5 @@ def task_cleanup_history() -> None:
     """
     clean up task queue
     """
+    from ws_bom_robot_app.task_manager import task_manager
     task_manager.cleanup_task()

ws_bom_robot_app/llm/vector_store/loader/base.py

@@ -113,6 +113,38 @@ class Loader():
 
   #@timer
   async def load(self) -> list[Document]:
+    #region log
+    import warnings
+    warnings.filterwarnings("ignore", message=".*pin_memory.*no accelerator is found.*")
+    log_msg_to_ignore = [
+        "Going to convert document batch...",
+        "Initializing pipeline for",
+        "Accelerator device:",
+        "detected formats:",
+    ]
+    class MessageFilter(logging.Filter):
+        def __init__(self, patterns):
+            super().__init__()
+            self.log_msg_to_ignore = patterns
+
+        def filter(self, record):
+            for pattern in self.log_msg_to_ignore:
+                if pattern in record.getMessage():
+                    return False
+            return True
+    message_filter = MessageFilter(log_msg_to_ignore)
+    loggers_to_filter = [
+        'docling',
+        'docling.document_converter',
+        'docling.datamodel',
+        'docling.datamodel.document',
+        'docling.utils.accelerator_utils',
+        'unstructured'
+    ]
+    for logger_name in loggers_to_filter:
+        logging.getLogger(logger_name).addFilter(message_filter)
+    #endregion log
+
     MAX_RETRIES = 3
     loaders: MergedDataLoader = MergedDataLoader(self.__directory_loader())
     try:
@@ -132,5 +164,8 @@ class Loader():
         finally:
            del _documents
     finally:
+      # Remove logging filters
+      for logger_name in loggers_to_filter:
+          logging.getLogger(logger_name).removeFilter(message_filter)
       del loaders
       gc.collect()

ws_bom_robot_app/main.py

@@ -1,157 +1,156 @@
 import datetime
-import platform
-from fastapi.responses import FileResponse
-import uvicorn, os, sys
-from fastapi import FastAPI, Depends
-from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
-from fastapi.openapi.utils import get_openapi
-from ws_bom_robot_app.auth import authenticate
-from ws_bom_robot_app.config import config
-from ws_bom_robot_app.llm.api import router as llm
-from ws_bom_robot_app.task_manager import router as task
-from ws_bom_robot_app.cron_manager import (
-    router as cron,
-    cron_manager)
-from ws_bom_robot_app.util import _log
-
+from fastapi import FastAPI
+from ws_bom_robot_app.util import is_app_subprocess
 _uptime = datetime.datetime.now()
-cron_manager.start()
 app = FastAPI(redoc_url=None,docs_url=None,openapi_url=None)
-app.include_router(llm,dependencies=[Depends(authenticate)])
-app.include_router(task,dependencies=[Depends(authenticate)])
-app.include_router(cron,dependencies=[Depends(authenticate)])
 
-@app.get("/")
-async def root():
-    return health()
-@app.get("/favicon.ico")
-async def favicon():
-    return FileResponse("./favicon.ico")
+if not is_app_subprocess():
+  import platform
+  from fastapi.responses import FileResponse
+  import os, sys
+  from fastapi import Depends
+  from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
+  from fastapi.openapi.utils import get_openapi
+  from ws_bom_robot_app.auth import authenticate
+  from ws_bom_robot_app.config import config
+  from ws_bom_robot_app.util import _log
+  from ws_bom_robot_app.llm.api import router as llm
+  from ws_bom_robot_app.task_manager import router as task
+  from ws_bom_robot_app.cron_manager import (
+      router as cron,
+      cron_manager)
+  cron_manager.start()
+  app.include_router(llm,dependencies=[Depends(authenticate)])
+  app.include_router(task,dependencies=[Depends(authenticate)])
+  app.include_router(cron,dependencies=[Depends(authenticate)])
 
-@app.get("/docs", include_in_schema=False)
-async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
-    return get_swagger_ui_html(openapi_url="/openapi.json", title="docs")
-@app.get("/redoc", include_in_schema=False)
-async def get_redoc_documentation(authenticate: bool = Depends(authenticate)):
-    return get_redoc_html(openapi_url="/openapi.json", title="docs")
-@app.get("/openapi.json", include_in_schema=False)
-async def openapi(authenticate: bool = Depends(authenticate)):
-    return get_openapi(title=app.title, version=app.version, routes=app.routes)
+  @app.get("/")
+  async def root():
+      return health()
+  @app.get("/favicon.ico")
+  async def favicon():
+      return FileResponse("./favicon.ico")
 
-@app.get("/api/health",tags=["diag"])
-def health():
-    return {"status": "ok"}
-def __get_size(bytes, suffix="B"):
-    """
-    Scale bytes to its proper format
-    e.g:
-        1253656 => '1.20MB'
-        1253656678 => '1.17GB'
-    """
-    factor = 1024
-    for unit in ["", "K", "M", "G", "T", "P"]:
-        if bytes < factor:
-            return f"{bytes:.2f}{unit}{suffix}"
-        bytes /= factor
-def __get_disk_info():
-    import psutil
-    partitions = psutil.disk_partitions()
-    _disks:list = []
-    for partition in partitions:
-        device = partition.device
-        mountpoint = partition.mountpoint
-        fstype = partition.fstype
-        try:
-            usage = psutil.disk_usage(mountpoint)
-        except PermissionError:
-            continue
-        total = __get_size(usage.total)
-        used = __get_size(usage.used)
-        free = __get_size(usage.free)
-        percent = f"{usage.percent}%"
-        _disks.append({"device": device, "mountpoint": mountpoint, "fstype": fstype, "total": total, "used": used, "free": free, "percent": percent})
-    return _disks
-@app.get("/api/diag",tags=["diag"])
-def diag(authenticate: bool = Depends(authenticate)):
-    import importlib,psutil
-    from ws_bom_robot_app.llm.providers.llm_manager import LlmManager as wsllm
-    from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager as wsdb
-    from ws_bom_robot_app.llm.vector_store.loader.base import Loader as wsldr
-    from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager as wsim
-    from ws_bom_robot_app.llm.tools.tool_manager import ToolManager as wstm
-    from ws_bom_robot_app.llm.agent_description import AgentDescriptor as wsad
+  @app.get("/docs", include_in_schema=False)
+  async def get_swagger_documentation(authenticate: bool = Depends(authenticate)):
+      return get_swagger_ui_html(openapi_url="/openapi.json", title="docs")
+  @app.get("/redoc", include_in_schema=False)
+  async def get_redoc_documentation(authenticate: bool = Depends(authenticate)):
+      return get_redoc_html(openapi_url="/openapi.json", title="docs")
+  @app.get("/openapi.json", include_in_schema=False)
+  async def openapi(authenticate: bool = Depends(authenticate)):
+      return get_openapi(title=app.title, version=app.version, routes=app.routes)
 
-    svmem = psutil.virtual_memory()
-    swap = psutil.swap_memory()
-    try:
-      ws_bom_robot_app_version = importlib.metadata.version("ws_bom_robot_app")
-    except:
-      ws_bom_robot_app_version = "unknown"
-    peer_process_ids = [c.pid for c in psutil.Process(os.getppid()).children()] if config.runtime_options().is_multi_process else None
-    return {
-        "status":"ok",
-        "uptime": {'from':_uptime,'elapsed':str(datetime.datetime.now()-_uptime)},
-        "system": {
-            "platform": {
-                "node": platform.node(),
-                "system": platform.system(),
-                "platform": platform.platform(),
-                "version": platform.version(),
-                "type": platform.machine(),
-                "processor": platform.processor(),
-                "architecture": platform.architecture()
-            },
-            "cpu": {
-                "physical_core": psutil.cpu_count(logical=False),
-                "total_core": psutil.cpu_count(logical=True),
-                "load": f"{psutil.cpu_percent(interval=1)}%"
-            },
-            "memory": {
-                "total": f"{__get_size(svmem.total)}",
-                "available": f"{__get_size(svmem.available)}",
-                "used": f"{__get_size(svmem.used)}",
-                "free": f"{__get_size(svmem.free)}",
-                "percent": f"{svmem.percent}%"
-            },
-            "swap": {
-                "total": f"{__get_size(swap.total)}",
-                "used": f"{__get_size(swap.used)}",
-                "free": f"{__get_size(swap.free)}",
-                "percent": f"{swap.percent}%"
-            },
-            "disk": __get_disk_info(),
-            "sys": {
-                "version": sys.version,
-                "platform": sys.platform,
-                "executable": sys.executable,
-                "args": {k: arg for k, arg in enumerate(sys.argv)}
-            },
-            "os": {
-                "ppid": os.getppid(),
-                "pid": os.getpid(),
-                "pids": peer_process_ids,
-                "cwd": os.getcwd(),
-                "ws_bom_robot_app": ws_bom_robot_app_version,
-                "env": os.environ,
-            },
-        },
-        "config":config,
-        "runtime":config.runtime_options(),
-        "extension": {
-            "provider":({item[0]: type(item[1]).__name__} for item in wsllm._list.items()),
-            "db":({item[0]: type(item[1]).__name__} for item in wsdb._list.items()),
-            "loader": ({item[0]: item[1].loader.__name__ if item[1] else None} for item in sorted(wsldr._list.items(), key=lambda x: x[0]) if item[1]),
-            "integration":({item[0]: type(item[1]).__name__} for item in wsim._list.items()),
-            "tool": ({item[0]: item[1].function.__name__} for item in wstm._list.items()),
-            "agent":({item[0]: type(item[1]).__name__} for item in wsad._list.items())
-            }
-        }
-@app.post("/diag/reload",tags=["diag"])
-def reset(authenticate: bool = Depends(authenticate)):
-    _log.info("restart server")
-    with open(".reloadfile","w") as f:
-        f.write("")
+  @app.get("/api/health",tags=["diag"])
+  def health():
+      return {"status": "ok"}
+  def __get_size(bytes, suffix="B"):
+      """
+      Scale bytes to its proper format
+      e.g:
+          1253656 => '1.20MB'
+          1253656678 => '1.17GB'
+      """
+      factor = 1024
+      for unit in ["", "K", "M", "G", "T", "P"]:
+          if bytes < factor:
+              return f"{bytes:.2f}{unit}{suffix}"
+          bytes /= factor
+  def __get_disk_info():
+      import psutil
+      partitions = psutil.disk_partitions()
+      _disks:list = []
+      for partition in partitions:
+          device = partition.device
+          mountpoint = partition.mountpoint
+          fstype = partition.fstype
+          try:
+              usage = psutil.disk_usage(mountpoint)
+          except PermissionError:
+              continue
+          total = __get_size(usage.total)
+          used = __get_size(usage.used)
+          free = __get_size(usage.free)
+          percent = f"{usage.percent}%"
+          _disks.append({"device": device, "mountpoint": mountpoint, "fstype": fstype, "total": total, "used": used, "free": free, "percent": percent})
+      return _disks
+  @app.get("/api/diag",tags=["diag"])
+  def diag(authenticate: bool = Depends(authenticate)):
+      import importlib,psutil
+      from ws_bom_robot_app.llm.providers.llm_manager import LlmManager as wsllm
+      from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager as wsdb
+      from ws_bom_robot_app.llm.vector_store.loader.base import Loader as wsldr
+      from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager as wsim
+      from ws_bom_robot_app.llm.tools.tool_manager import ToolManager as wstm
+      from ws_bom_robot_app.llm.agent_description import AgentDescriptor as wsad
 
-# Start the FastAPI server
-if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=6001, env_file="../.env", reload=True, log_level="debug")
+      svmem = psutil.virtual_memory()
+      swap = psutil.swap_memory()
+      try:
+        ws_bom_robot_app_version = importlib.metadata.version("ws_bom_robot_app")
+      except:
+        ws_bom_robot_app_version = "unknown"
+      peer_process_ids = [c.pid for c in psutil.Process(os.getppid()).children()] if config.runtime_options().is_multi_process else None
+      return {
+          "status":"ok",
+          "uptime": {'from':_uptime,'elapsed':str(datetime.datetime.now()-_uptime)},
+          "system": {
+              "platform": {
+                  "node": platform.node(),
+                  "system": platform.system(),
+                  "platform": platform.platform(),
+                  "version": platform.version(),
+                  "type": platform.machine(),
+                  "processor": platform.processor(),
+                  "architecture": platform.architecture()
+              },
+              "cpu": {
+                  "physical_core": psutil.cpu_count(logical=False),
+                  "total_core": psutil.cpu_count(logical=True),
+                  "load": f"{psutil.cpu_percent(interval=1)}%"
+              },
+              "memory": {
+                  "total": f"{__get_size(svmem.total)}",
+                  "available": f"{__get_size(svmem.available)}",
+                  "used": f"{__get_size(svmem.used)}",
+                  "free": f"{__get_size(svmem.free)}",
+                  "percent": f"{svmem.percent}%"
+              },
+              "swap": {
+                  "total": f"{__get_size(swap.total)}",
+                  "used": f"{__get_size(swap.used)}",
+                  "free": f"{__get_size(swap.free)}",
+                  "percent": f"{swap.percent}%"
+              },
+              "disk": __get_disk_info(),
+              "sys": {
+                  "version": sys.version,
+                  "platform": sys.platform,
+                  "executable": sys.executable,
+                  "args": {k: arg for k, arg in enumerate(sys.argv)}
+              },
+              "os": {
+                  "ppid": os.getppid(),
+                  "pid": os.getpid(),
+                  "pids": peer_process_ids,
+                  "cwd": os.getcwd(),
+                  "ws_bom_robot_app": ws_bom_robot_app_version,
+                  "env": os.environ,
+              },
+          },
+          "config":config,
+          "runtime":config.runtime_options(),
+          "extension": {
+              "provider":({item[0]: type(item[1]).__name__} for item in wsllm._list.items()),
+              "db":({item[0]: type(item[1]).__name__} for item in wsdb._list.items()),
+              "loader": ({item[0]: item[1].loader.__name__ if item[1] else None} for item in sorted(wsldr._list.items(), key=lambda x: x[0]) if item[1]),
+              "integration":({item[0]: type(item[1]).__name__} for item in wsim._list.items()),
+              "tool": ({item[0]: item[1].function.__name__} for item in wstm._list.items()),
+              "agent":({item[0]: type(item[1]).__name__} for item in wsad._list.items())
+              }
+          }
+  @app.post("/diag/reload",tags=["diag"])
+  def reset(authenticate: bool = Depends(authenticate)):
+      _log.info("restart server")
+      with open(".reloadfile","w") as f:
+          f.write("")

ws_bom_robot_app/subprocess_runner.py

@@ -12,6 +12,9 @@ def _worker_run_pickled(serialized_task: bytes, conn: Connection):
     capture return value or exception and send back via conn.send((ok_flag, payload_serialized)).
     This runs in a separate process and must be top-level for multiprocessing.
     """
+    import os
+    # mark as a subprocess
+    os.environ['IS_ROBOT_APP_SUBPROCESS'] = 'true'
     try:
         if _pickler is None:
             raise RuntimeError("No pickler available in worker process.")

ws_bom_robot_app/util.py

@@ -22,6 +22,12 @@ def logger_instance(name: str) -> logging.Logger:
 _log: logging.Logger = locals().get("_loc", logger_instance(__name__))
 #endregion
 
+#region task
+def is_app_subprocess():
+    """Check if we're running a task in a subprocess."""
+    return os.environ.get('IS_ROBOT_APP_SUBPROCESS', '').lower() == 'true'
+#endregion
+
 #region cache
 _cache = {}
 _cache_timestamps = {}

{ws_bom_robot_app-0.0.84.dist-info → ws_bom_robot_app-0.0.85.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.84
+Version: 0.0.85
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.84.dist-info → ws_bom_robot_app-0.0.85.dist-info}/RECORD

@@ -1,11 +1,11 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
 ws_bom_robot_app/config.py,sha256=CASD6fCCBp9YODBdlJWGN0vw7__hwrbq8WjrrLVRbTg,5307
-ws_bom_robot_app/cron_manager.py,sha256=HU_0CGqR7YUyI9MJ5CMTLp-s-A6HgdZCIFeV7SoGKvo,9361
-ws_bom_robot_app/main.py,sha256=LZH4z9BmVlxpFJf8TrIo_JxH1YhpeZRrrOYgKky7S7w,6712
-ws_bom_robot_app/subprocess_runner.py,sha256=sq06r3sdHc_1FdPcpN9Frf2-xKu7dM76FK9x8J7I0kc,4100
+ws_bom_robot_app/cron_manager.py,sha256=TOz7dsQhbGXzYMKW7GboKOSySg9aun4h0yLckj-5w4U,9372
+ws_bom_robot_app/main.py,sha256=5h4qwQ4Ghm6CCSjO5eWvMhWxDATzUayQfQ-__E1Mw1I,6936
+ws_bom_robot_app/subprocess_runner.py,sha256=N71HxPvgMP5TIRlO5w9UzHAEK-JKOA9i16QXM3anpjM,4195
 ws_bom_robot_app/task_manager.py,sha256=jaxRnMCVMlxQzHyhNrt6duH4ov1zblf3-Sv8cwmesyI,24039
-ws_bom_robot_app/util.py,sha256=RjVD6B9sHje788Lndqq5DHy6TJM0KLs9qx3JYt81Wyk,4834
+ws_bom_robot_app/util.py,sha256=t1VS6JQNOZe6aenBmjPLxJ_A3ncm7WqTZE8_gR85sQo,5022
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_context.py,sha256=uatHJ8wcRly6h0S762BgfzDMpmcwCHwNzwo37aWjeE0,1305
 ws_bom_robot_app/llm/agent_description.py,sha256=yK4aVU3RNk1oP4bEneV3QPAi-208JwWk4R6qHlzqYIg,4656
@@ -33,7 +33,7 @@ ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaN
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent.py,sha256=_CY5Dji3UeAIi2iuU7ttz4fml1q8aCFgVWOv970x8Fw,1411
 ws_bom_robot_app/llm/utils/chunker.py,sha256=zVXjRMloc3KbNEqiDcycYzy4N0Ey1g8XYeq6ftyvkyg,857
-ws_bom_robot_app/llm/utils/cleanup.py,sha256=0V-4CNucOIDlw7wu8zd-0_M4Xu5AVkPokCyuoys49qM,3059
+ws_bom_robot_app/llm/utils/cleanup.py,sha256=ARLZTX4mLbkLCEnMdIWYDYEAPOjzfy1laLGkYnxZe30,3063
 ws_bom_robot_app/llm/utils/cms.py,sha256=XhrLQyHQ2JUOInDCCf_uvR4Jiud0YvH2FwwiiuCnnsg,6352
 ws_bom_robot_app/llm/utils/download.py,sha256=yBrw9n6lbz1QlWhApIlEwuQ8kMa3u11OFXx84X_NRvA,7130
 ws_bom_robot_app/llm/utils/print.py,sha256=IsPYEWRJqu-dqlJA3F9OnnIS4rOq_EYX1Ljp3BvDnww,774
@@ -65,10 +65,10 @@ ws_bom_robot_app/llm/vector_store/integration/sitemap.py,sha256=FJy2wQDvML_1fR4g
 ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=hiE1kkg7868mbP2wVWQLmC1fK2jIE1lT7f8hVN0NqeY,2636
 ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=PylagYLzhSY_wMu_hR4PzAwSm4Jp6zi2aymuF0XN4Hw,4271
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=b__KxrYql5j6I6vqO43gnEocN43Ic3o1OpTfV2n8pko,5656
+ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=GjUS2oaz0LHOSal5pipBkomZtrYUNcKPSd8bzhUU5Dc,6889
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=IOv1A0HSIWiHWQFzI4fdApfxrKgXOqwmC3mPXlKplqQ,4012
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.84.dist-info/METADATA,sha256=voKK2OL7tpgb8zEIHK8zUBFSJQE8-5aQYFFAWousmd0,9971
-ws_bom_robot_app-0.0.84.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.84.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.84.dist-info/RECORD,,
+ws_bom_robot_app-0.0.85.dist-info/METADATA,sha256=md1udg5NIma6BagHw38GcOjmQp2tOq5fqHvToFAqvFw,9971
+ws_bom_robot_app-0.0.85.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ws_bom_robot_app-0.0.85.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.85.dist-info/RECORD,,