ws-bom-robot-app 0.0.83__py3-none-any.whl → 0.0.84__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -23,6 +23,8 @@ class Settings(BaseSettings):
     robot_task_strategy: str = 'memory' # memory / db
     robot_task_mp_enable: bool = True
     robot_task_mp_method: str = 'spawn' # spawn / fork
+    robot_task_mp_max_retries: int = 1
+    robot_task_mp_retry_delay: float = 60 # seconds
     robot_cron_strategy: str = 'memory' # memory / db
     robot_cms_host: str = ''
     robot_cms_auth: str = ''

ws_bom_robot_app/llm/utils/chunker.py

@@ -1,12 +1,17 @@
 from langchain_core.documents import Document
 from langchain_text_splitters import CharacterTextSplitter
+import logging
 
 class DocumentChunker:
+  _MAX_CHUNK_SIZE = 10_000
   @staticmethod
   def chunk(documents: list[Document]) -> list[Document]:
-      text_splitter = CharacterTextSplitter(chunk_size=10_000, chunk_overlap=500)
+      text_splitter = CharacterTextSplitter(chunk_size=DocumentChunker._MAX_CHUNK_SIZE, chunk_overlap=int(DocumentChunker._MAX_CHUNK_SIZE * 0.02))
       chunked_documents = []
       for doc in documents:
+          if len(doc.page_content) <= DocumentChunker._MAX_CHUNK_SIZE:
+              chunked_documents.append(doc)
+              continue
           chunks = text_splitter.split_text(doc.page_content)
           for chunk in chunks:
               chunked_documents.append(

ws_bom_robot_app/llm/vector_store/db/base.py

@@ -50,8 +50,8 @@ class VectorDBStrategy(ABC):
           Asynchronously invokes multiple retrievers in parallel, then merges
           their results while removing duplicates.
       """
+    MAX_TOKENS_PER_BATCH = 300_000 * 0.8
     def __init__(self):
-        self.max_tokens_per_batch = 300_000 * 0.8  # conservative limit below 300k openai limit: https://platform.openai.com/docs/api-reference/embeddings/create
         try:
             self.encoding = tiktoken.get_encoding("cl100k_base")  # text-embedding-3-small, text-embedding-3-large: https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken
         except Exception:
@@ -78,7 +78,7 @@ class VectorDBStrategy(ABC):
       for doc in documents:
           doc_tokens = self._count_tokens(doc.page_content)
           # check if adding this document exceeds the limit
-          if current_token_count + doc_tokens > self.max_tokens_per_batch:
+          if current_token_count + doc_tokens > VectorDBStrategy.MAX_TOKENS_PER_BATCH:
               # start new batch if current batch is not empty
               if current_batch:
                   batches.append(current_batch)

ws_bom_robot_app/llm/vector_store/loader/docling.py

@@ -6,6 +6,7 @@ from langchain_core.runnables import run_in_executor
 from docling.document_converter import DocumentConverter, InputFormat, PdfFormatOption, ImageFormatOption
 from docling.datamodel.pipeline_options import PdfPipelineOptions, TableStructureOptions, TableFormerMode, TesseractCliOcrOptions
 from langchain_community.document_loaders import UnstructuredFileLoader
+from ws_bom_robot_app.llm.vector_store.db.base import VectorDBStrategy
 
 class DoclingLoader(BaseLoader):
   def __init__(self, file_path: str | list[str], **kwargs: Any) -> None:
@@ -37,28 +38,37 @@ class DoclingLoader(BaseLoader):
           if doc is done:
               break
           yield doc  # type: ignore[misc]
+  def _fallback_loader(self, source: str, error: Exception = None) -> Iterator[Document]:
+      if 'fallback' in self._kwargs:
+          if issubclass(self._kwargs['fallback'], (BaseLoader, UnstructuredFileLoader)):
+            logging.info(f"Using fallback loader {self._kwargs['fallback']} for {source}")
+            try:
+              loader: Union[BaseLoader, UnstructuredFileLoader] = self._kwargs['fallback'](
+                  source,
+                  **{k: v for k, v in self._kwargs.items() if k != 'fallback'}
+                  )
+              yield from loader.lazy_load()
+            except Exception as e:
+              logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")
+          else:
+              logging.warning(f"Invalid fallback loader {self._kwargs['fallback']}[{type(self._kwargs['fallback'])}] for {source}")
+      else:
+        logging.warning(f"Failed to load document from {source}: {error}")
   def lazy_load(self) -> Iterator[Document]:
       for source in self._file_paths:
           try:
-            _result = self._converter.convert(
-               os.path.abspath(source),
-               raises_on_error=True)
-            doc = _result.document
-            text = doc.export_to_markdown(image_placeholder="")
-            yield Document(page_content=text, metadata={"source": source})
-          except Exception as e:
-            if 'fallback' in self._kwargs:
-                if issubclass(self._kwargs['fallback'], (BaseLoader, UnstructuredFileLoader)):
-                  logging.info(f"Using fallback loader {self._kwargs['fallback']} for {source}")
-                  try:
-                    loader: Union[BaseLoader, UnstructuredFileLoader] = self._kwargs['fallback'](
-                       source,
-                       **{k: v for k, v in self._kwargs.items() if k != 'fallback'}
-                       )
-                    yield from loader.lazy_load()
-                  except Exception as e:
-                    logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")
-                else:
-                   logging.warning(f"Invalid fallback loader {self._kwargs['fallback']}[{type(self._kwargs['fallback'])}] for {source}")
+            #manage only small file with header, preventing header stripping and improper chunking
+            if (source.endswith('.csv') or source.endswith('.xlsx')) \
+                and 'fallback' in self._kwargs \
+                and os.path.getsize(source) > (VectorDBStrategy.MAX_TOKENS_PER_BATCH // 4): #rough token estimate
+              yield from self._fallback_loader(source)
             else:
-              logging.warning(f"Failed to load document from {source}: {e} | {traceback.format_exc()}")
+              _result = self._converter.convert(
+                os.path.abspath(source),
+                raises_on_error=True)
+              doc = _result.document
+              text = doc.export_to_markdown(image_placeholder="")
+              yield Document(page_content=text, metadata={"source": source})
+          except Exception as e:
+             yield from self._fallback_loader(source,e)
+

ws_bom_robot_app/task_manager.py

@@ -69,6 +69,7 @@ class TaskStatus(IdentifiableEntity):
     result: Optional[T] = None
     metadata: TaskMetaData = None
     error: Optional[str] = None
+    retry: int = 0
     model_config = ConfigDict(
         arbitrary_types_allowed=True
     )
@@ -118,7 +119,8 @@ class TaskEntry(IdentifiableEntity):
 class TaskStatistics(BaseModel):
     class TaskStatisticExecutionInfo(BaseModel):
         retention_days: float = config.robot_task_retention_days
-        max_concurrent: int
+        max_parallelism: int
+        slot_available: int
         pid: int = os.getpid()
         running: list[TaskStatus]
         slowest: list
@@ -137,9 +139,12 @@ class TaskStatistics(BaseModel):
 
 #region interface
 class TaskManagerStrategy(ABC):
-    def __init__(self, max_concurrent_tasks: int = max(1,floor(config.robot_task_max_total_parallelism / config.runtime_options().number_of_workers))):
-        self.max_concurrent_tasks = max_concurrent_tasks
-        self.semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
+    def __init__(self, max_concurrent_tasks: Optional[int] = None):
+        if max_concurrent_tasks is None:
+            workers = config.runtime_options().number_of_workers
+            max_concurrent_tasks = max(1, floor(config.robot_task_max_total_parallelism / max(1, workers)))
+        self.max_parallelism = max_concurrent_tasks
+        self.semaphore = asyncio.Semaphore(max_concurrent_tasks)
         self.running_tasks = dict[str, TaskEntry]()
         self.loop = asyncio.get_event_loop()
 
@@ -201,14 +206,15 @@ class TaskManagerStrategy(ABC):
           #remove from running tasks
           if task_entry.id in self.running_tasks:
               del self.running_tasks[task_entry.id]
-          #notify webhooks
-          if task_entry.headers and task_entry.headers.x_ws_bom_webhooks:
-            try:
-                asyncio.create_task(
-                    WebhookNotifier().notify_webhook(task_entry.status, task_entry.headers.x_ws_bom_webhooks)
-                )
-            except Exception as e:
-                _log.error(f"Failed to schedule webhook notification for task {task_entry.id}: {e}")
+          #notify webhooks: a task has completed or failed, if failed with retry policy the task remains in pending state, and will not be notified until complete/failure
+          if task_entry.status.status in ["completed","failure"]:
+            if task_entry.headers and task_entry.headers.x_ws_bom_webhooks:
+              try:
+                  asyncio.create_task(
+                      WebhookNotifier().notify_webhook(task_entry.status, task_entry.headers.x_ws_bom_webhooks)
+                  )
+              except Exception as e:
+                  _log.error(f"Failed to schedule webhook notification for task {task_entry.id}: {e}")
 
     def task_done_callback(self, task_entry: TaskEntry) -> Callable:
         def callback(task: asyncio.Task, context: Any | None = None):
@@ -260,6 +266,55 @@ class TaskManagerStrategy(ABC):
         async with self.semaphore:
           await self._execute_task(task_entry)
 
+    async def _monitor_subprocess(self, task_entry: TaskEntry, proc, conn):
+        try:
+            # Wait for the worker to send bytes (this blocks, so run via executor wrapper)
+            data_bytes = await _recv_from_connection_async(conn)
+            # unpickle bytes to get payload
+            try:
+                payload = _pickler.loads(data_bytes)
+            except Exception:
+                # fallback if pickler fails
+                payload = ("err", {"error": "Failed to unpickle subprocess result"})
+            if isinstance(payload, tuple) and payload[0] == "ok":
+                result = payload[1]
+                # write results into task_entry
+                self._update_task_by_event(task_entry, "completed", result)
+            else:
+                # error
+                err_info = payload[1]["error"] if isinstance(payload, tuple) else str(payload)
+                self._update_task_by_event(task_entry, "failure", err_info) # give up, no retry
+        except Exception:
+            # maybe subprocess is no more alive / killed due to memory pressure
+            if task_entry.status.retry < config.robot_task_mp_max_retries:
+                task_entry.status.retry += 1
+                _log.warning(f"Task {task_entry.id} failure, retrying {task_entry.status.retry}...")
+                async def delayed_retry():
+                    _delay = config.robot_task_mp_retry_delay # help to backpressure when overloaded
+                    if self.semaphore._value > 0:  # free semaphore slots available
+                        _delay = 5  # small/no delay if retry can run immediately
+                    await asyncio.sleep(_delay)  # delay in seconds
+                    await self._run_task_with_semaphore(task_entry)
+                asyncio.create_task(delayed_retry())
+                # semaphore is released, so new task can be executed
+                return
+            else:
+                self._update_task_by_event(task_entry, "failure", "subprocess monitor error: failed to receive data from connection")
+        finally:
+            # ensure process termination / cleanup
+            try:
+                conn.close()
+            except Exception:
+                pass
+            try:
+              if proc.is_alive():
+                  proc.terminate()
+              proc.join(timeout=1)
+            except Exception:
+                pass
+            # callback
+            self._update_task_by_event(task_entry, "callback", None)
+
     async def _execute_task(self, task_entry: TaskEntry):
         """
         Execute the task. Try to run it inside a subprocess (if serializable).
@@ -269,56 +324,19 @@ class TaskManagerStrategy(ABC):
         """
         self.running_tasks[task_entry.id]=task_entry
         task_entry.status.metadata.start_at = str(datetime.now().isoformat())
-        # Try to spawn subprocess (non-blocking)
+        # try to spawn subprocess (non-blocking)
         can_use_subprocess = task_entry.status.metadata.extra.get("can_use_subprocess", False)
         if config.robot_task_mp_enable and can_use_subprocess:
             proc, conn, used_subprocess = _start_subprocess_for_coroutine(task_entry.coroutine)
             if used_subprocess and proc is not None and conn is not None:
-                # We will monitor the subprocess asynchronously
+                # monitor subprocess asynchronously
                 task_entry.status.status = "pending"
                 task_entry.status.metadata.pid_child = proc.pid
                 _log.info(f"Task {task_entry.id} started in subprocess (pid={proc.pid})")
-
-            async def _monitor_subprocess():
-                try:
-                    # Wait for the worker to send bytes (this blocks, so run via executor wrapper)
-                    data_bytes = await _recv_from_connection_async(conn)
-                    # unpickle bytes to get payload
-                    try:
-                        payload = _pickler.loads(data_bytes)
-                    except Exception:
-                        # fallback if pickler fails
-                        payload = ("err", {"error": "Failed to unpickle subprocess result"})
-                    if isinstance(payload, tuple) and payload[0] == "ok":
-                        result = payload[1]
-                        # write results into task_entry
-                        self._update_task_by_event(task_entry, "completed", result)
-                    else:
-                        # error
-                        err_info = payload[1]["error"] if isinstance(payload, tuple) else str(payload)
-                        self._update_task_by_event(task_entry, "failure", err_info)
-                except Exception:
-                    # maybe subprocess is no more alive / killed
-                    self._update_task_by_event(task_entry, "failure", "subprocess monitor error: failed to receive data from connection")
-                finally:
-                    # Ensure process termination / cleanup
-                    try:
-                        conn.close()
-                    except Exception:
-                        pass
-                    try:
-                      if proc.is_alive():
-                          proc.terminate()
-                      proc.join(timeout=1)
-                    except Exception:
-                        pass
-                    # callback
-                    self._update_task_by_event(task_entry, "callback", None)
-
-            # schedule monitor task and return
-            asyncio.create_task(_monitor_subprocess())
-            return
-        # fallback
+                # await monitor process, then return: important to acquire semaphore
+                await self._monitor_subprocess(task_entry, proc, conn)
+                return
+        # default fallback (in-process)
         try:
             async def _callable_to_coroutine(func: Any) -> Any:
                 if callable(func) and not inspect.iscoroutine(func):
@@ -373,7 +391,8 @@ class TaskManagerStrategy(ABC):
             ),
             exec_info=TaskStatistics.TaskStatisticExecutionInfo(
                 retention_days=config.robot_task_retention_days,
-                max_concurrent=self.max_concurrent_tasks,
+                max_parallelism=self.max_parallelism,
+                slot_available=self.semaphore._value,
                 running=[task.status for task in self.running_task()],
                 slowest=_slowest
             )
@@ -383,8 +402,8 @@ class TaskManagerStrategy(ABC):
 
 #region memory implementation
 class MemoryTaskManagerStrategy(TaskManagerStrategy):
-    def __init__(self):
-        super().__init__()
+    def __init__(self, max_concurrent_tasks: Optional[int] = None):
+        super().__init__(max_concurrent_tasks)
         self.tasks: Dict[str, TaskEntry] = {}
 
     def create_task(self, coroutine: Any, headers: TaskHeader | None = None) -> IdentifiableEntity:
@@ -425,8 +444,8 @@ class TaskEntryModel(Base):
         arbitrary_types_allowed=True
     )
 class DatabaseTaskManagerStrategy(TaskManagerStrategy):
-    def __init__(self, db_url: str = f"sqlite:///{config.robot_data_folder}/db/tasks.sqlite"):
-        super().__init__()
+    def __init__(self, db_url: str = f"sqlite:///{config.robot_data_folder}/db/tasks.sqlite", max_concurrent_tasks: Optional[int] = None):
+        super().__init__(max_concurrent_tasks)
         self.engine = create_engine(db_url)
         self.Session = sessionmaker(bind=self.engine)
         Base.metadata.create_all(self.engine)

{ws_bom_robot_app-0.0.83.dist-info → ws_bom_robot_app-0.0.84.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.83
+Version: 0.0.84
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.83.dist-info → ws_bom_robot_app-0.0.84.dist-info}/RECORD

@@ -1,10 +1,10 @@
 ws_bom_robot_app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/auth.py,sha256=84nIbmJsMrNs0sxIQGEHbjsjc2P6ZrZZGSn8dkiL6is,895
-ws_bom_robot_app/config.py,sha256=sNoJyxxZM7xTmqMdojjyjrlmV_7Pctl8ZKRNju_VtoE,5214
+ws_bom_robot_app/config.py,sha256=CASD6fCCBp9YODBdlJWGN0vw7__hwrbq8WjrrLVRbTg,5307
 ws_bom_robot_app/cron_manager.py,sha256=HU_0CGqR7YUyI9MJ5CMTLp-s-A6HgdZCIFeV7SoGKvo,9361
 ws_bom_robot_app/main.py,sha256=LZH4z9BmVlxpFJf8TrIo_JxH1YhpeZRrrOYgKky7S7w,6712
 ws_bom_robot_app/subprocess_runner.py,sha256=sq06r3sdHc_1FdPcpN9Frf2-xKu7dM76FK9x8J7I0kc,4100
-ws_bom_robot_app/task_manager.py,sha256=lxCrF07kwmrPgam5dNZDzqRPrPC4m9GxJWfvNeyhenI,22748
+ws_bom_robot_app/task_manager.py,sha256=jaxRnMCVMlxQzHyhNrt6duH4ov1zblf3-Sv8cwmesyI,24039
 ws_bom_robot_app/util.py,sha256=RjVD6B9sHje788Lndqq5DHy6TJM0KLs9qx3JYt81Wyk,4834
 ws_bom_robot_app/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/agent_context.py,sha256=uatHJ8wcRly6h0S762BgfzDMpmcwCHwNzwo37aWjeE0,1305
@@ -32,7 +32,7 @@ ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent.py,sha256=_CY5Dji3UeAIi2iuU7ttz4fml1q8aCFgVWOv970x8Fw,1411
-ws_bom_robot_app/llm/utils/chunker.py,sha256=N7570xBYlObneg-fsvDhPAJ-Pv8C8OaYZOBK6q7LmMI,607
+ws_bom_robot_app/llm/utils/chunker.py,sha256=zVXjRMloc3KbNEqiDcycYzy4N0Ey1g8XYeq6ftyvkyg,857
 ws_bom_robot_app/llm/utils/cleanup.py,sha256=0V-4CNucOIDlw7wu8zd-0_M4Xu5AVkPokCyuoys49qM,3059
 ws_bom_robot_app/llm/utils/cms.py,sha256=XhrLQyHQ2JUOInDCCf_uvR4Jiud0YvH2FwwiiuCnnsg,6352
 ws_bom_robot_app/llm/utils/download.py,sha256=yBrw9n6lbz1QlWhApIlEwuQ8kMa3u11OFXx84X_NRvA,7130
@@ -42,7 +42,7 @@ ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIw
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/generator.py,sha256=9_xdtCKJhmt1OP0GXDjvFERXMP7ozLZT92KuYEBDgC0,6314
 ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/db/base.py,sha256=t0Z1VCcg604evEzJENGNqYFBi_AZLTEUzmxA5wgoE_A,8419
+ws_bom_robot_app/llm/vector_store/db/base.py,sha256=iQBY-1O8uJ1_dRzmCDmhWAMBYtpims0u5RelQMvOVLo,8310
 ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=2riMQvwe2T99X_NtO9yO9lpZ0zj2Nb06l9Hb1lWJ00E,4509
 ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=Y2LpMsU0Ce2RCaGM1n69BxMpXWXpBoj1T5aAAJpX2qE,3860
 ws_bom_robot_app/llm/vector_store/db/manager.py,sha256=5rqBvc0QKmHFUgVHqBAr1Y4FZRl-w-ylGMjgXZywrdA,533
@@ -66,9 +66,9 @@ ws_bom_robot_app/llm/vector_store/integration/slack.py,sha256=hiE1kkg7868mbP2wVW
 ws_bom_robot_app/llm/vector_store/integration/thron.py,sha256=PylagYLzhSY_wMu_hR4PzAwSm4Jp6zi2aymuF0XN4Hw,4271
 ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=b__KxrYql5j6I6vqO43gnEocN43Ic3o1OpTfV2n8pko,5656
-ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=TPLtqr4oG4tHZwv9db5KRb3vx4ZPo9Ez6mGpgC4dVkg,3477
+ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=IOv1A0HSIWiHWQFzI4fdApfxrKgXOqwmC3mPXlKplqQ,4012
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=qo9ejRZyKv_k6jnGgXnu1W5uqsMMtgqK_uvPpZQ0p74,833
-ws_bom_robot_app-0.0.83.dist-info/METADATA,sha256=bBfHufKR-l-Aap7_nPrHAjP7jzEBD1VfM30XwzdMOak,9971
-ws_bom_robot_app-0.0.83.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.83.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.83.dist-info/RECORD,,
+ws_bom_robot_app-0.0.84.dist-info/METADATA,sha256=voKK2OL7tpgb8zEIHK8zUBFSJQE8-5aQYFFAWousmd0,9971
+ws_bom_robot_app-0.0.84.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ws_bom_robot_app-0.0.84.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.84.dist-info/RECORD,,