ws-bom-robot-app 0.0.81__tar.gz → 0.0.83__tar.gz

{ws_bom_robot_app-0.0.81/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.83}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.81
+Version: 0.0.83
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -15,7 +15,7 @@ Requires-Dist: apscheduler==3.11.0
 Requires-Dist: aiofiles==24.1.0
 Requires-Dist: pydantic==2.11.7
 Requires-Dist: pydantic-settings==2.10.1
-Requires-Dist: fastapi[standard]==0.115.14
+Requires-Dist: fastapi[standard]==0.116.1
 Requires-Dist: chevron==0.14.0
 Requires-Dist: langchain==0.3.26
 Requires-Dist: langchain-community==0.3.26
@@ -34,9 +34,9 @@ Requires-Dist: fastembed==0.7.1
 Requires-Dist: langchain-qdrant==0.2.0
 Requires-Dist: qdrant-client==1.15.0
 Requires-Dist: lark==1.2.2
-Requires-Dist: unstructured==0.16.21
+Requires-Dist: unstructured==0.18.11
 Requires-Dist: unstructured[image]
-Requires-Dist: unstructured-ingest==0.5.4
+Requires-Dist: unstructured-ingest==1.2.6
 Requires-Dist: unstructured-ingest[azure]
 Requires-Dist: unstructured-ingest[confluence]
 Requires-Dist: unstructured-ingest[dropbox]
@@ -115,7 +115,7 @@ GOOGLE_APPLICATION_CREDENTIALS="./.data/secrets/google-credentials.json"
 
   ```bash
   fastapi dev --port 6001
-  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --host 0.0.0.0 --port 6001 
+  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --reload-dir ws_bom_robot_app --host 0.0.0.0 --port 6001 
   ```  
 
 - production
@@ -145,6 +145,8 @@ dockerize base image
 
 ```pwsh
 <# cpu #>
+#docker build -f Dockerfile-robot-base-cpu -t ws-bom-robot-base:cpu .
+#docker tag ws-bom-robot-base:cpu ghcr.io/websolutespa/ws-bom-robot-base:cpu
 docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
 docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
 <# gpu #>
@@ -152,17 +154,25 @@ docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-b
 docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
 ```
 
-dockerize app
+dockerize app (from src)
 
 ```pwsh
 docker build -f Dockerfile -t ws-bom-robot-app .
-docker run --rm --name ws-bom-robot-app -d --env-file .env -p 6001:6001 ws-bom-robot-app
+docker run --rm -d --name ws-bom-robot-app --env-file .env -p 6001:6001 ws-bom-robot-app
+```
+
+dockerize app (from latest)
+
+```pwsh
+docker build -f Dockerfile-pkg -t ws-bom-robot-app-pkg .
+docker run --rm -d --name ws-bom-robot-app-pkg --env-file .env -p 6001:6001 ws-bom-robot-app-pkg
 ```
 
 docker run mounted to src (dev mode)
 
 ```pwsh
-docker run --rm --name ws-bom-robot-app-src -d --env-file .env -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tmp:/tmp"  -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app uvicorn ws_bom_robot_app.main:app --reload --host 0.0.0.0 --port 6001
 ```
 
 ---
@@ -288,7 +298,7 @@ prospector ./ws_bom_robot_app -t pyroma
 # pyclean --verbose .
 pytest --cov=ws_bom_robot_app --log-cli-level=info
 # directory
-# pytest --cov=ws_bom_robot_app --log-cli-level=info ./tests/app/llm/vector_store/db
+# pytest --cov=ws_bom_robot_app.llm.vector_store --log-cli-level=info ./tests/app/llm/vector_store
 ```
 
 #### 🐞 start debugger

{ws_bom_robot_app-0.0.81 → ws_bom_robot_app-0.0.83}/README.md

@@ -49,7 +49,7 @@ GOOGLE_APPLICATION_CREDENTIALS="./.data/secrets/google-credentials.json"
 
   ```bash
   fastapi dev --port 6001
-  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --host 0.0.0.0 --port 6001 
+  #uvicorn main:app --app-dir ./ws_bom_robot_app --reload --reload-dir ws_bom_robot_app --host 0.0.0.0 --port 6001 
   ```  
 
 - production
@@ -79,6 +79,8 @@ dockerize base image
 
 ```pwsh
 <# cpu #>
+#docker build -f Dockerfile-robot-base-cpu -t ws-bom-robot-base:cpu .
+#docker tag ws-bom-robot-base:cpu ghcr.io/websolutespa/ws-bom-robot-base:cpu
 docker build -f Dockerfile-robot-base-cpu -t ghcr.io/websolutespa/ws-bom-robot-base:cpu .
 docker push ghcr.io/websolutespa/ws-bom-robot-base:cpu
 <# gpu #>
@@ -86,17 +88,25 @@ docker build -f Dockerfile-robot-base-gpu -t ghcr.io/websolutespa/ws-bom-robot-b
 docker push ghcr.io/websolutespa/ws-bom-robot-base:gpu
 ```
 
-dockerize app
+dockerize app (from src)
 
 ```pwsh
 docker build -f Dockerfile -t ws-bom-robot-app .
-docker run --rm --name ws-bom-robot-app -d --env-file .env -p 6001:6001 ws-bom-robot-app
+docker run --rm -d --name ws-bom-robot-app --env-file .env -p 6001:6001 ws-bom-robot-app
+```
+
+dockerize app (from latest)
+
+```pwsh
+docker build -f Dockerfile-pkg -t ws-bom-robot-app-pkg .
+docker run --rm -d --name ws-bom-robot-app-pkg --env-file .env -p 6001:6001 ws-bom-robot-app-pkg
 ```
 
 docker run mounted to src (dev mode)
 
 ```pwsh
-docker run --rm --name ws-bom-robot-app-src -d --env-file .env -v "$(pwd)/ws_bom_robot_app:/app/ws_bom_robot_app" -v "$(pwd)/.data:/app/.data" -v "$(pwd)/tmp:/tmp"  -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app fastapi dev ./ws_bom_robot_app/main.py --host 0.0.0.0 --port 6001
+docker run --rm  -d --env-file .env -v "$(pwd)/.data:/app/.data" -p 6001:6001 ws-bom-robot-app uvicorn ws_bom_robot_app.main:app --reload --host 0.0.0.0 --port 6001
 ```
 
 ---
@@ -222,7 +232,7 @@ prospector ./ws_bom_robot_app -t pyroma
 # pyclean --verbose .
 pytest --cov=ws_bom_robot_app --log-cli-level=info
 # directory
-# pytest --cov=ws_bom_robot_app --log-cli-level=info ./tests/app/llm/vector_store/db
+# pytest --cov=ws_bom_robot_app.llm.vector_store --log-cli-level=info ./tests/app/llm/vector_store
 ```
 
 #### 🐞 start debugger

{ws_bom_robot_app-0.0.81 → ws_bom_robot_app-0.0.83}/requirements.txt

@@ -4,7 +4,7 @@ apscheduler==3.11.0
 aiofiles==24.1.0
 pydantic==2.11.7
 pydantic-settings==2.10.1
-fastapi[standard]==0.115.14
+fastapi[standard]==0.116.1
 chevron==0.14.0
 
 #framework
@@ -29,9 +29,9 @@ qdrant-client==1.15.0
 lark==1.2.2 #self-query retriever
 
 #loaders
-unstructured==0.16.21
+unstructured==0.18.11
 unstructured[image]
-unstructured-ingest==0.5.4
+unstructured-ingest==1.2.6
 unstructured-ingest[azure]
 unstructured-ingest[confluence]
 unstructured-ingest[dropbox]

{ws_bom_robot_app-0.0.81 → ws_bom_robot_app-0.0.83}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.81",
+    version="0.0.83",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.81 → ws_bom_robot_app-0.0.83}/ws_bom_robot_app/config.py

@@ -16,9 +16,14 @@ class Settings(BaseSettings):
     robot_data_db_retention_days: float = 60
     robot_data_attachment_folder: str = 'attachment'
     robot_data_attachment_retention_days: float = 1
+    robot_ingest_max_threads: int = 1 # safe choice to 1, avoid potential process-related issues with Docker
     robot_loader_max_threads: int = 1
     robot_task_max_total_parallelism: int = 2 * (os.cpu_count() or 1)
     robot_task_retention_days: float = 1
+    robot_task_strategy: str = 'memory' # memory / db
+    robot_task_mp_enable: bool = True
+    robot_task_mp_method: str = 'spawn' # spawn / fork
+    robot_cron_strategy: str = 'memory' # memory / db
     robot_cms_host: str = ''
     robot_cms_auth: str = ''
     robot_cms_db_folder: str = 'llmVectorDb'
@@ -41,6 +46,7 @@ class Settings(BaseSettings):
     )
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        # env
         os.environ["USER_AGENT"] = self.USER_AGENT
         os.environ["OPENAI_API_KEY"] = self.OPENAI_API_KEY
         os.environ["OLLAMA_API_URL"] = self.OLLAMA_API_URL
@@ -53,6 +59,10 @@ class Settings(BaseSettings):
         os.environ["WATSONX_APIKEY"] = self.WATSONX_APIKEY
         os.environ["WATSONX_PROJECTID"] = self.WATSONX_PROJECTID
         os.environ["NEBULY_API_URL"] = self.NEBULY_API_URL
+        # dir
+        os.makedirs(self.robot_data_folder, exist_ok=True)
+        for subfolder in [self.robot_data_db_folder, self.robot_data_attachment_folder, 'db']:
+            os.makedirs(os.path.join(self.robot_data_folder, subfolder), exist_ok=True)
 
     class RuntimeOptions(BaseModel):
         @staticmethod

{ws_bom_robot_app-0.0.81 → ws_bom_robot_app-0.0.83}/ws_bom_robot_app/cron_manager.py

@@ -1,3 +1,4 @@
+import os
 from apscheduler.schedulers.background import BackgroundScheduler
 #from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.jobstores.memory import MemoryJobStore
@@ -7,8 +8,7 @@ from apscheduler.triggers.interval import IntervalTrigger
 from apscheduler.triggers.date import DateTrigger
 from fastapi import APIRouter
 from datetime import datetime
-from ws_bom_robot_app.task_manager import task_manager
-from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment
+from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment, task_cleanup_history
 from ws_bom_robot_app.util import _log
 from ws_bom_robot_app.config import config
 
@@ -22,8 +22,8 @@ class MemoryJobstoreStrategy(JobstoreStrategy):
         return {"default": MemoryJobStore()}
 
 class PersistentJobstoreStrategy(JobstoreStrategy):
-    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
-        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+    def get_jobstore(self, db_url: str = f"sqlite:///{config.robot_data_folder}/db/jobs.sqlite"):
+        _log.info(f"Using persistent cron jobstore with database URL: {db_url}.")
         return {"default": SQLAlchemyJobStore(url=db_url)}
 
 class Job:
@@ -56,12 +56,12 @@ class Job:
 
 class CronManager:
     _list_default = [
-            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
+            Job('cleanup-task-history',task_cleanup_history, interval=5 * 60),
             Job('cleanup-kb-data',kb_cleanup_data_file, interval=180 * 60),
             Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=120 * 60),
         ]
     def __get_jobstore_strategy(self) -> JobstoreStrategy:
-        if True or config.runtime_options().is_multi_process:
+        if config.robot_cron_strategy == 'memory':
             return MemoryJobstoreStrategy()
         return PersistentJobstoreStrategy()
     def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):

{ws_bom_robot_app-0.0.81 → ws_bom_robot_app-0.0.83}/ws_bom_robot_app/llm/agent_description.py

@@ -1,123 +1,123 @@
-import json, requests, re
-from typing import Any
-from abc import ABC, abstractmethod
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.messages import AIMessage
-from langchain_core.runnables import RunnableSerializable
-from langchain_core.runnables import RunnableLambda
-from bs4 import BeautifulSoup
-from ws_bom_robot_app.llm.models.api import LlmRules
-from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
-from ws_bom_robot_app.llm.utils.agent import get_rules
-
-# SafeDict helper class
-class SafeDict(dict):
-    def __missing__(self, key):
-        return ''
-
-# Strategy Interface
-class AgentDescriptorStrategy(ABC):
-    @abstractmethod
-    def enrich_prompt(self, prompt: str, input: dict) -> str:
-        pass
-
-    @abstractmethod
-    def rule_input(self, input: dict) -> str:
-        pass
-
-# Concrete Strategy for Default Agent
-class DefaultAgentDescriptor(AgentDescriptorStrategy):
-    def enrich_prompt(self, prompt: str, input: dict) -> str:
-        # Default enrichment logic (could be minimal or no-op)
-        return prompt.format_map(SafeDict(input))
-
-    def rule_input(self, input: dict) -> str:
-        return input.get('content', "")
-
-# Concrete Strategy for URL2Text Agent
-class URL2TextAgentDescriptor(AgentDescriptorStrategy):
-    def enrich_prompt(self, prompt: str, input: dict) -> str:
-        input["context"] = self._get_page_text(input)
-        return prompt.format_map(SafeDict(input))
-
-    def rule_input(self, input: dict) -> str:
-        return input.get('context', "")
-
-    def _get_page_text(self, input: dict) -> str:
-        url = input.get("content", "")
-        exclusions = input.get("exclude", {})
-        response = requests.get(url)
-        response.raise_for_status()
-        soup = BeautifulSoup(response.content, 'html5lib')
-        classes_to_exclude = exclusions.get("classes", [])
-        ids_to_exclude = exclusions.get("ids", [])
-        for class_name in classes_to_exclude:
-            for element in soup.find_all(class_=class_name):
-                element.extract()
-        for id_name in ids_to_exclude:
-            for element in soup.find_all(id=id_name):
-                element.extract()
-        for script in soup(["script", "noscript", "style", "head", "footer", "iframe"]):
-            script.extract()
-        return re.sub(' +', ' ', soup.get_text())
-
-
-class AgentDescriptor:
-    # Dictionary to hold all agent strategies
-    _list: dict[str,AgentDescriptorStrategy] = {
-        "default": DefaultAgentDescriptor(),
-        "url2text": URL2TextAgentDescriptor(),
-    }
-
-    # Functions to manage strategies
-    @staticmethod
-    def add_strategy(name: str, strategy: AgentDescriptorStrategy):
-        """_summary_
-            add a new strategy to the dictionary
-        Args:
-            name (str): name of the strategy, in lowercase
-            strategy (AgentDescriptorStrategy): class implementing the strategy
-        Examples:
-            AgentDescriptor.add_strategy("custom_agent_descriptor", CustomAgentDescriptor())
-        """
-        AgentDescriptor._list[name.lower()] = strategy
-
-    @staticmethod
-    def get_strategy(name: str) -> AgentDescriptorStrategy:
-        return AgentDescriptor._list.get(name.lower(), DefaultAgentDescriptor())
-
-    def __init__(self, llm: LlmInterface, prompt: str, mode: str, rules: LlmRules = None):
-        self.__prompt = prompt
-        self.__llm = llm
-        self.rules= rules
-        self.strategy = self.get_strategy(mode)  # Selects the strategy from the dictionary
-
-    async def __create_prompt(self, input_dict: dict):
-        input_data = json.loads(input_dict.get("input", {}))
-        system = self.strategy.enrich_prompt(self.__prompt, input_data)
-        if self.rules:
-            rule_input = self.strategy.rule_input(input_data)
-            rules_prompt = await get_rules(self.__llm.get_embeddings(), self.rules, rule_input)
-            system += rules_prompt
-        return ChatPromptTemplate.from_messages(
-            [
-                ("system", system),
-                ("user", input_data.get("content", ""))
-            ]
-        )
-
-    def __create_agent_descriptor(self, content) -> RunnableSerializable[Any, Any]:
-        content = json.loads(content)
-        agent = (
-            {
-                "input": lambda x: x["input"],
-            }
-            | RunnableLambda(self.__create_prompt)
-            | self.__llm.get_llm()
-        )
-        return agent
-
-    async def run_agent(self, content) -> Any:
-        agent_descriptor = self.__create_agent_descriptor(content)
-        response: AIMessage = await agent_descriptor.ainvoke({"input": content})
-        return response
+import json, requests, re
+from typing import Any
+from abc import ABC, abstractmethod
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.messages import AIMessage
+from langchain_core.runnables import RunnableSerializable
+from langchain_core.runnables import RunnableLambda
+from bs4 import BeautifulSoup
+from ws_bom_robot_app.llm.models.api import LlmRules
+from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
+from ws_bom_robot_app.llm.utils.agent import get_rules
+
+# SafeDict helper class
+class SafeDict(dict):
+    def __missing__(self, key):
+        return ''
+
+# Strategy Interface
+class AgentDescriptorStrategy(ABC):
+    @abstractmethod
+    def enrich_prompt(self, prompt: str, input: dict) -> str:
+        pass
+
+    @abstractmethod
+    def rule_input(self, input: dict) -> str:
+        pass
+
+# Concrete Strategy for Default Agent
+class DefaultAgentDescriptor(AgentDescriptorStrategy):
+    def enrich_prompt(self, prompt: str, input: dict) -> str:
+        # Default enrichment logic (could be minimal or no-op)
+        return prompt.format_map(SafeDict(input))
+
+    def rule_input(self, input: dict) -> str:
+        return input.get('content', "")
+
+# Concrete Strategy for URL2Text Agent
+class URL2TextAgentDescriptor(AgentDescriptorStrategy):
+    def enrich_prompt(self, prompt: str, input: dict) -> str:
+        input["context"] = self._get_page_text(input)
+        return prompt.format_map(SafeDict(input))
+
+    def rule_input(self, input: dict) -> str:
+        return input.get('context', "")
+
+    def _get_page_text(self, input: dict) -> str:
+        url = input.get("content", "")
+        exclusions = input.get("exclude", {})
+        response = requests.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html5lib')
+        classes_to_exclude = exclusions.get("classes", [])
+        ids_to_exclude = exclusions.get("ids", [])
+        for class_name in classes_to_exclude:
+            for element in soup.find_all(class_=class_name):
+                element.extract()
+        for id_name in ids_to_exclude:
+            for element in soup.find_all(id=id_name):
+                element.extract()
+        for script in soup(["script", "noscript", "style", "head", "footer", "iframe"]):
+            script.extract()
+        return re.sub(' +', ' ', soup.get_text())
+
+
+class AgentDescriptor:
+    # Dictionary to hold all agent strategies
+    _list: dict[str,AgentDescriptorStrategy] = {
+        "default": DefaultAgentDescriptor(),
+        "url2text": URL2TextAgentDescriptor(),
+    }
+
+    # Functions to manage strategies
+    @staticmethod
+    def add_strategy(name: str, strategy: AgentDescriptorStrategy):
+        """_summary_
+            add a new strategy to the dictionary
+        Args:
+            name (str): name of the strategy, in lowercase
+            strategy (AgentDescriptorStrategy): class implementing the strategy
+        Examples:
+            AgentDescriptor.add_strategy("custom_agent_descriptor", CustomAgentDescriptor())
+        """
+        AgentDescriptor._list[name.lower()] = strategy
+
+    @staticmethod
+    def get_strategy(name: str) -> AgentDescriptorStrategy:
+        return AgentDescriptor._list.get(name.lower(), DefaultAgentDescriptor())
+
+    def __init__(self, llm: LlmInterface, prompt: str, mode: str, rules: LlmRules = None):
+        self.__prompt = prompt
+        self.__llm = llm
+        self.rules= rules
+        self.strategy = self.get_strategy(mode)  # Selects the strategy from the dictionary
+
+    async def __create_prompt(self, input_dict: dict):
+        input_data = json.loads(input_dict.get("input", {}))
+        system = self.strategy.enrich_prompt(self.__prompt, input_data)
+        if self.rules:
+            rule_input = self.strategy.rule_input(input_data)
+            rules_prompt = await get_rules(self.__llm.get_embeddings(), self.rules, rule_input)
+            system += rules_prompt
+        return ChatPromptTemplate.from_messages(
+            [
+                ("system", system),
+                ("user", input_data.get("content", ""))
+            ]
+        )
+
+    def __create_agent_descriptor(self, content) -> RunnableSerializable[Any, Any]:
+        content = json.loads(content)
+        agent = (
+            {
+                "input": lambda x: x["input"],
+            }
+            | RunnableLambda(self.__create_prompt)
+            | self.__llm.get_llm()
+        )
+        return agent
+
+    async def run_agent(self, content) -> Any:
+        agent_descriptor = self.__create_agent_descriptor(content)
+        response: AIMessage = await agent_descriptor.ainvoke({"input": content})
+        return response