ws-bom-robot-app 0.0.37__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/config.py

@@ -4,6 +4,7 @@ from pydantic_settings import BaseSettings
 import os
 
 class Settings(BaseSettings):
+    USER_AGENT: str = 'ws-bom-robot'
     robot_env: str = 'local'
     robot_user: str = 'user'
     robot_password: str = 'password'
@@ -13,19 +14,34 @@ class Settings(BaseSettings):
     robot_data_db_folder_out: str = 'out'
     robot_data_db_folder_store: str = 'store'
     robot_data_db_retention_days: float = 60
+    robot_data_attachment_folder: str = 'attachment'
+    robot_data_attachment_retention_days: float = 1
+    robot_ingest_max_threads: int = 1 # safe choice to 1, avoid potential process-related issues with Docker
     robot_loader_max_threads: int = 1
     robot_task_max_total_parallelism: int = 2 * (os.cpu_count() or 1)
     robot_task_retention_days: float = 1
+    robot_task_strategy: str = 'memory' # memory / db
+    robot_task_mp_enable: bool = True
+    robot_task_mp_method: str = 'spawn' # spawn / fork
+    robot_task_mp_max_retries: int = 1
+    robot_task_mp_retry_delay: float = 60 # seconds
+    robot_cron_strategy: str = 'memory' # memory / db
     robot_cms_host: str = ''
     robot_cms_auth: str = ''
     robot_cms_db_folder: str = 'llmVectorDb'
     robot_cms_kb_folder: str ='llmKbFile'
     ANTHROPIC_API_KEY: str = ''
+    DEEPSEEK_API_KEY: str = ''
     OPENAI_API_KEY: str = '' # used also for saas dall-e api
     OLLAMA_API_URL: str = 'http://localhost:11434'
     GROQ_API_KEY: str = ''
     GOOGLE_API_KEY: str = ''
     GOOGLE_APPLICATION_CREDENTIALS: str = '' # path to google credentials iam file, e.d. ./.secrets/google-credentials.json
+    WATSONX_URL: str = ''
+    WATSONX_APIKEY: str = ''
+    WATSONX_PROJECTID: str = ''
+    NEBULY_API_URL: str ='https://backend.nebuly.com/'
+    LANGSMITH_API_KEY: str = '' # app-wide api key to run evaluation
     model_config = ConfigDict(
         env_file='./.env',
         extra='ignore',
@@ -33,16 +49,28 @@ class Settings(BaseSettings):
     )
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        # env
+        os.environ["USER_AGENT"] = self.USER_AGENT
         os.environ["OPENAI_API_KEY"] = self.OPENAI_API_KEY
         os.environ["OLLAMA_API_URL"] = self.OLLAMA_API_URL
         os.environ["ANTHROPIC_API_KEY"] = self.ANTHROPIC_API_KEY
+        os.environ["DEEPSEEK_API_KEY"] = self.DEEPSEEK_API_KEY
         os.environ["GROQ_API_KEY"] = self.GROQ_API_KEY
         os.environ["GOOGLE_API_KEY"] = self.GOOGLE_API_KEY
         os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.GOOGLE_APPLICATION_CREDENTIALS
+        os.environ["WATSONX_URL"] = self.WATSONX_URL
+        os.environ["WATSONX_APIKEY"] = self.WATSONX_APIKEY
+        os.environ["WATSONX_PROJECTID"] = self.WATSONX_PROJECTID
+        os.environ["NEBULY_API_URL"] = self.NEBULY_API_URL
+        os.environ["LANGSMITH_API_KEY"] = self.LANGSMITH_API_KEY
+        # dir
+        os.makedirs(self.robot_data_folder, exist_ok=True)
+        for subfolder in [self.robot_data_db_folder, self.robot_data_attachment_folder, 'db']:
+            os.makedirs(os.path.join(self.robot_data_folder, subfolder), exist_ok=True)
 
     class RuntimeOptions(BaseModel):
         @staticmethod
-        def _get_number_of_workers() -> int:
+        def _get_sys_arg(arg: str, default: int) -> int:
             """
             Returns the number of worker processes to use for the application.
 
@@ -60,18 +88,18 @@ class Settings(BaseSettings):
             """
             import sys
             try:
-                for i, arg in enumerate(sys.argv):
-                    if arg == "--workers" and i + 1 < len(sys.argv):
+                for i, argv in enumerate(sys.argv):
+                    if argv == f"--{arg}" and i + 1 < len(sys.argv):
                         return int(sys.argv[i + 1])
             except (ValueError, IndexError):
                 pass
-            return 1
+            return default
         debug: bool
+        tcp_port: int = _get_sys_arg("port", 6001)
         loader_show_progress: bool
         loader_silent_errors: bool
-        number_of_workers: int = _get_number_of_workers()
-        is_multi_process: bool = _get_number_of_workers() > 1
-
+        number_of_workers: int = _get_sys_arg("workers", 1)
+        is_multi_process: bool = _get_sys_arg("workers", 1) > 1
 
     def runtime_options(self) -> RuntimeOptions:
       """_summary_

ws_bom_robot_app/cron_manager.py

@@ -1,3 +1,4 @@
+import os
 from apscheduler.schedulers.background import BackgroundScheduler
 #from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.jobstores.memory import MemoryJobStore
@@ -7,8 +8,7 @@ from apscheduler.triggers.interval import IntervalTrigger
 from apscheduler.triggers.date import DateTrigger
 from fastapi import APIRouter
 from datetime import datetime
-from ws_bom_robot_app.task_manager import task_manager
-from ws_bom_robot_app.llm.utils.kb import kb_cleanup_data_file
+from ws_bom_robot_app.llm.utils.cleanup import kb_cleanup_data_file, chat_cleanup_attachment, task_cleanup_history
 from ws_bom_robot_app.util import _log
 from ws_bom_robot_app.config import config
 
@@ -22,8 +22,8 @@ class MemoryJobstoreStrategy(JobstoreStrategy):
         return {"default": MemoryJobStore()}
 
 class PersistentJobstoreStrategy(JobstoreStrategy):
-    def get_jobstore(self, db_url: str = "sqlite:///.data/db/jobs.sqlite"):
-        _log.info(f"Using persistent crob jobstore with database URL: {db_url}.")
+    def get_jobstore(self, db_url: str = f"sqlite:///{config.robot_data_folder}/db/jobs.sqlite"):
+        _log.info(f"Using persistent cron jobstore with database URL: {db_url}.")
         return {"default": SQLAlchemyJobStore(url=db_url)}
 
 class Job:
@@ -56,17 +56,18 @@ class Job:
 
 class CronManager:
     _list_default = [
-            Job('cleanup-task',task_manager.cleanup_task, interval=5 * 60),
-            Job('cleanup-data',kb_cleanup_data_file, interval=180 * 60),
+            Job('cleanup-task-history',task_cleanup_history, interval=4 * 60 * 60),
+            Job('cleanup-kb-data',kb_cleanup_data_file, interval=8 * 60 * 60),
+            Job('cleanup-chat-attachment',chat_cleanup_attachment, interval=6 * 60 * 60),
         ]
-    def __get_jobstore_strategy() -> JobstoreStrategy:
-        if True or config.runtime_options().is_multi_process:
+    def __get_jobstore_strategy(self) -> JobstoreStrategy:
+        if config.robot_cron_strategy == 'memory':
             return MemoryJobstoreStrategy()
         return PersistentJobstoreStrategy()
     def __init__(self, strategy: JobstoreStrategy = None, enable_defaults: bool = True):
         self.enable_defaults = enable_defaults
         if strategy is None:
-          strategy = CronManager.__get_jobstore_strategy()
+          strategy = self.__get_jobstore_strategy()
         jobstores = strategy.get_jobstore()
         self.scheduler: BackgroundScheduler = BackgroundScheduler(jobstores=jobstores)
         self.__scheduler_is_running = False
@@ -139,22 +140,22 @@ class CronManager:
 
     def execute_recurring_jobs(self):
         for job in self.scheduler.get_jobs():
-            if job.interval:
-                job.job_func()
+            if job.trigger.interval:
+                job.func()
 
     def pause_recurring_jobs(self):
         for job in self.scheduler.get_jobs():
-            if job.interval:
+            if job.trigger.interval:
                 self.pause_job(job.id)
 
     def resume_recurring_jobs(self):
         for job in self.scheduler.get_jobs():
-            if job.interval:
+            if job.trigger.interval:
                 self.resume_job(job.id)
 
     def remove_recurring_jobs(self):
         for job in self.scheduler.get_jobs():
-            if job.interval:
+            if job.trigger.interval:
                 self.remove_job(job.id)
 
     def clear(self):

ws_bom_robot_app/llm/agent_context.py

@@ -0,0 +1,26 @@
+import uuid
+from datetime import datetime
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field
+from typing import Optional
+
+class AgentContext(BaseModel):
+    class _i18n(BaseModel):
+        lg: Optional[str] = "en"
+        country: Optional[str] = "US"
+        timestamp: Optional[str] = Field(default_factory=lambda: datetime.now().strftime("%Y-%m-%d %H:%M:%S %A"))
+        timezone: Optional[str] = "UTC"
+        model_config = ConfigDict(extra='allow')
+    class _user(BaseModel):
+        id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+        first_name: Optional[str] = Field(None, validation_alias=AliasChoices("firstName","first_name"))
+        last_name: Optional[str] = Field(None, validation_alias=AliasChoices("lastName","last_name"))
+        country: Optional[str] = ''
+        email: Optional[str] = ''
+        phone: Optional[str] = ''
+        role: Optional[list] = [] #i.e. ["admin","user","guest"]
+        department: Optional[list] = [] #i.e. ["R&D","IT","HR"]
+        permission: Optional[list] = [] #i.e. ["read","write","delete","execute"]
+        model_config = ConfigDict(extra='allow')
+    i18n: _i18n = Field(default_factory=_i18n)
+    user: Optional[_user] =Field(default_factory=_user)
+    model_config = ConfigDict(extra='allow')

ws_bom_robot_app/llm/agent_description.py

@@ -1,123 +1,123 @@
-import json, requests, re
-from typing import Any
-from abc import ABC, abstractmethod
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.messages import AIMessage
-from langchain_core.runnables import RunnableSerializable
-from langchain_core.runnables import RunnableLambda
-from bs4 import BeautifulSoup
-from ws_bom_robot_app.llm.models.api import LlmRules
-from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
-from ws_bom_robot_app.llm.utils.agent_utils import get_rules
-
-# SafeDict helper class
-class SafeDict(dict):
-    def __missing__(self, key):
-        return ''
-
-# Strategy Interface
-class AgentDescriptorStrategy(ABC):
-    @abstractmethod
-    def enrich_prompt(self, prompt: str, input: dict) -> str:
-        pass
-
-    @abstractmethod
-    def rule_input(self, input: dict) -> str:
-        pass
-
-# Concrete Strategy for Default Agent
-class DefaultAgentDescriptor(AgentDescriptorStrategy):
-    def enrich_prompt(self, prompt: str, input: dict) -> str:
-        # Default enrichment logic (could be minimal or no-op)
-        return prompt.format_map(SafeDict(input))
-
-    def rule_input(self, input: dict) -> str:
-        return input.get('content', "")
-
-# Concrete Strategy for URL2Text Agent
-class URL2TextAgentDescriptor(AgentDescriptorStrategy):
-    def enrich_prompt(self, prompt: str, input: dict) -> str:
-        input["context"] = self._get_page_text(input)
-        return prompt.format_map(SafeDict(input))
-
-    def rule_input(self, input: dict) -> str:
-        return input.get('context', "")
-
-    def _get_page_text(self, input: dict) -> str:
-        url = input.get("content", "")
-        exclusions = input.get("exclude", {})
-        response = requests.get(url)
-        response.raise_for_status()
-        soup = BeautifulSoup(response.content, 'html5lib')
-        classes_to_exclude = exclusions.get("classes", [])
-        ids_to_exclude = exclusions.get("ids", [])
-        for class_name in classes_to_exclude:
-            for element in soup.find_all(class_=class_name):
-                element.extract()
-        for id_name in ids_to_exclude:
-            for element in soup.find_all(id=id_name):
-                element.extract()
-        for script in soup(["script", "noscript", "style", "head", "footer", "iframe"]):
-            script.extract()
-        return re.sub(' +', ' ', soup.get_text())
-
-
-class AgentDescriptor:
-    # Dictionary to hold all agent strategies
-    _list: dict[str,AgentDescriptorStrategy] = {
-        "default": DefaultAgentDescriptor(),
-        "url2text": URL2TextAgentDescriptor(),
-    }
-
-    # Functions to manage strategies
-    @staticmethod
-    def add_strategy(name: str, strategy: AgentDescriptorStrategy):
-        """_summary_
-            add a new strategy to the dictionary
-        Args:
-            name (str): name of the strategy, in lowercase
-            strategy (AgentDescriptorStrategy): class implementing the strategy
-        Examples:
-            AgentDescriptor.add_strategy("custom_agent_descriptor", CustomAgentDescriptor())
-        """
-        AgentDescriptor._list[name.lower()] = strategy
-
-    @staticmethod
-    def get_strategy(name: str) -> AgentDescriptorStrategy:
-        return AgentDescriptor._list.get(name.lower(), DefaultAgentDescriptor())
-
-    def __init__(self, llm: LlmInterface, prompt: str, mode: str, rules: LlmRules = None):
-        self.__prompt = prompt
-        self.__llm = llm,
-        self.rules= rules
-        self.strategy = self.get_strategy(mode)  # Selects the strategy from the dictionary
-
-    async def __create_prompt(self, input_dict: dict):
-        input_data = json.loads(input_dict.get("input", {}))
-        system = self.strategy.enrich_prompt(self.__prompt, input_data)
-        if self.rules:
-            rule_input = self.strategy.rule_input(input_data)
-            rules_prompt = await get_rules(self.__llm.get_embeddings(), self.rules, rule_input)
-            system += rules_prompt
-        return ChatPromptTemplate.from_messages(
-            [
-                ("system", system),
-                ("user", input_data.get("content", ""))
-            ]
-        )
-
-    def __create_agent_descriptor(self, content) -> RunnableSerializable[Any, Any]:
-        content = json.loads(content)
-        agent = (
-            {
-                "input": lambda x: x["input"],
-            }
-            | RunnableLambda(self.__create_prompt)
-            | self.__llm.get_llm()
-        )
-        return agent
-
-    async def run_agent(self, content) -> Any:
-        agent_descriptor = self.__create_agent_descriptor(content)
-        response: AIMessage = await agent_descriptor.ainvoke({"input": content})
-        return response
+import json, requests, re
+from typing import Any
+from abc import ABC, abstractmethod
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.messages import AIMessage
+from langchain_core.runnables import RunnableSerializable
+from langchain_core.runnables import RunnableLambda
+from bs4 import BeautifulSoup
+from ws_bom_robot_app.llm.models.api import LlmRules
+from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
+from ws_bom_robot_app.llm.utils.agent import get_rules
+
+# SafeDict helper class
+class SafeDict(dict):
+    def __missing__(self, key):
+        return ''
+
+# Strategy Interface
+class AgentDescriptorStrategy(ABC):
+    @abstractmethod
+    def enrich_prompt(self, prompt: str, input: dict) -> str:
+        pass
+
+    @abstractmethod
+    def rule_input(self, input: dict) -> str:
+        pass
+
+# Concrete Strategy for Default Agent
+class DefaultAgentDescriptor(AgentDescriptorStrategy):
+    def enrich_prompt(self, prompt: str, input: dict) -> str:
+        # Default enrichment logic (could be minimal or no-op)
+        return prompt.format_map(SafeDict(input))
+
+    def rule_input(self, input: dict) -> str:
+        return input.get('content', "")
+
+# Concrete Strategy for URL2Text Agent
+class URL2TextAgentDescriptor(AgentDescriptorStrategy):
+    def enrich_prompt(self, prompt: str, input: dict) -> str:
+        input["context"] = self._get_page_text(input)
+        return prompt.format_map(SafeDict(input))
+
+    def rule_input(self, input: dict) -> str:
+        return input.get('context', "")
+
+    def _get_page_text(self, input: dict) -> str:
+        url = input.get("content", "")
+        exclusions = input.get("exclude", {})
+        response = requests.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html5lib')
+        classes_to_exclude = exclusions.get("classes", [])
+        ids_to_exclude = exclusions.get("ids", [])
+        for class_name in classes_to_exclude:
+            for element in soup.find_all(class_=class_name):
+                element.extract()
+        for id_name in ids_to_exclude:
+            for element in soup.find_all(id=id_name):
+                element.extract()
+        for script in soup(["script", "noscript", "style", "head", "footer", "iframe"]):
+            script.extract()
+        return re.sub(' +', ' ', soup.get_text())
+
+
+class AgentDescriptor:
+    # Dictionary to hold all agent strategies
+    _list: dict[str,AgentDescriptorStrategy] = {
+        "default": DefaultAgentDescriptor(),
+        "url2text": URL2TextAgentDescriptor(),
+    }
+
+    # Functions to manage strategies
+    @staticmethod
+    def add_strategy(name: str, strategy: AgentDescriptorStrategy):
+        """_summary_
+            add a new strategy to the dictionary
+        Args:
+            name (str): name of the strategy, in lowercase
+            strategy (AgentDescriptorStrategy): class implementing the strategy
+        Examples:
+            AgentDescriptor.add_strategy("custom_agent_descriptor", CustomAgentDescriptor())
+        """
+        AgentDescriptor._list[name.lower()] = strategy
+
+    @staticmethod
+    def get_strategy(name: str) -> AgentDescriptorStrategy:
+        return AgentDescriptor._list.get(name.lower(), DefaultAgentDescriptor())
+
+    def __init__(self, llm: LlmInterface, prompt: str, mode: str, rules: LlmRules = None):
+        self.__prompt = prompt
+        self.__llm = llm
+        self.rules= rules
+        self.strategy = self.get_strategy(mode)  # Selects the strategy from the dictionary
+
+    async def __create_prompt(self, input_dict: dict):
+        input_data = json.loads(input_dict.get("input", {}))
+        system = self.strategy.enrich_prompt(self.__prompt, input_data)
+        if self.rules:
+            rule_input = self.strategy.rule_input(input_data)
+            rules_prompt = await get_rules(self.__llm.get_embeddings(), self.rules, rule_input)
+            system += rules_prompt
+        return ChatPromptTemplate.from_messages(
+            [
+                ("system", system),
+                ("user", input_data.get("content", ""))
+            ]
+        )
+
+    def __create_agent_descriptor(self, content) -> RunnableSerializable[Any, Any]:
+        content = json.loads(content)
+        agent = (
+            {
+                "input": lambda x: x["input"],
+            }
+            | RunnableLambda(self.__create_prompt)
+            | self.__llm.get_llm()
+        )
+        return agent
+
+    async def run_agent(self, content) -> Any:
+        agent_descriptor = self.__create_agent_descriptor(content)
+        response: AIMessage = await agent_descriptor.ainvoke({"input": content})
+        return response