ws-bom-robot-app 0.0.57__tar.gz → 0.0.59__tar.gz

{ws_bom_robot_app-0.0.57/ws_bom_robot_app.egg-info → ws_bom_robot_app-0.0.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.57
+Version: 0.0.59
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -17,15 +17,16 @@ Requires-Dist: pydantic==2.10.6
 Requires-Dist: pydantic-settings==2.7.1
 Requires-Dist: fastapi[standard]==0.115.8
 Requires-Dist: chevron==0.14.0
-Requires-Dist: langchain==0.3.18
-Requires-Dist: langchain-community==0.3.17
-Requires-Dist: langchain-core==0.3.34
-Requires-Dist: langchain-openai==0.3.5
+Requires-Dist: trafilatura==2.0.0
+Requires-Dist: langchain==0.3.25
+Requires-Dist: langchain-community==0.3.24
+Requires-Dist: langchain-core==0.3.59
+Requires-Dist: langchain-openai==0.3.16
 Requires-Dist: langchain-anthropic==0.3.6
 Requires-Dist: langchain-google-genai==2.0.7
 Requires-Dist: langchain-google-vertexai==2.0.13
-Requires-Dist: langchain-groq==0.2.4
-Requires-Dist: langchain-ollama==0.2.3
+Requires-Dist: langchain-groq==0.3.2
+Requires-Dist: langchain-ollama==0.3.2
 Requires-Dist: faiss-cpu==1.9.0
 Requires-Dist: chromadb==0.6.3
 Requires-Dist: langchain_chroma==0.2.1
@@ -48,6 +49,7 @@ Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
+Requires-Dist: duckduckgo-search==8.0.4
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/requirements.txt

@@ -6,17 +6,18 @@ pydantic==2.10.6
 pydantic-settings==2.7.1
 fastapi[standard]==0.115.8
 chevron==0.14.0
+trafilatura==2.0.0
 
 #framework
-langchain==0.3.18
-langchain-community==0.3.17
-langchain-core==0.3.34
-langchain-openai==0.3.5
+langchain==0.3.25
+langchain-community==0.3.24
+langchain-core==0.3.59
+langchain-openai==0.3.16
 langchain-anthropic==0.3.6 #issue get_models() from 0.3.7
 langchain-google-genai==2.0.7 #waiting for new release: https://github.com/langchain-ai/langchain-google/issues/711
 langchain-google-vertexai==2.0.13
-langchain-groq==0.2.4
-langchain-ollama==0.2.3
+langchain-groq==0.3.2
+langchain-ollama==0.3.2
 
 #vector DB
 faiss-cpu==1.9.0
@@ -46,3 +47,4 @@ html5lib==1.1 #beautifulsoup4 parser
 
 #integrations
 markdownify==0.14.1 #sitemap
+duckduckgo-search==8.0.4

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/setup.py

@@ -4,7 +4,7 @@ _requirements = [line.split('#')[0].strip() for line in open("requirements.txt")
 
 setup(
     name="ws_bom_robot_app",
-    version="0.0.57",
+    version="0.0.59",
     description="A FastAPI application serving ws bom/robot/llm platform ai.",
     long_description=open("README.md", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/ws_bom_robot_app/config.py

@@ -28,6 +28,7 @@ class Settings(BaseSettings):
     GOOGLE_API_KEY: str = ''
     NEBULY_API_URL: str =''
     GOOGLE_APPLICATION_CREDENTIALS: str = '' # path to google credentials iam file, e.d. ./.secrets/google-credentials.json
+    TAVILY_API_KEY: str = '' #TODO DELETE
     model_config = ConfigDict(
         env_file='./.env',
         extra='ignore',
@@ -43,6 +44,8 @@ class Settings(BaseSettings):
         os.environ["GOOGLE_API_KEY"] = self.GOOGLE_API_KEY
         os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.GOOGLE_APPLICATION_CREDENTIALS
         os.environ["NEBULY_API_URL"] = self.NEBULY_API_URL
+        # TODO DELETE
+        os.environ["TAVILY_API_KEY"] = self.TAVILY_API_KEY
 
     class RuntimeOptions(BaseModel):
         @staticmethod

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/ws_bom_robot_app/llm/agent_handler.py

@@ -41,7 +41,6 @@ class AgentHandler(AsyncCallbackHandler):
         self.json_start_regex = re.compile(r'(`{1,3}\s*json\b)') # detect a potential json start fence.
         self.json_end_regex = re.compile(r'(`{1,3})')         # an end fence (one to three backticks).
         self.stream_cut_last_output_chunk_size = 16  # safe cut last chunk size to output if no markers are found
-
     async def on_chat_model_start(self, serialized, messages, *, run_id, parent_run_id = None, tags = None, metadata = None, **kwargs):
         if not self.__started:
           self.__started = True
@@ -102,7 +101,7 @@ class AgentHandler(AsyncCallbackHandler):
                       finally:
                           self.json_buffer = ""
                       # remove the end fence from pending.
-                      self.stream_buffer = self.stream_buffer[end_match.end():].replace('`','').strip()
+                      self.stream_buffer = self.stream_buffer[end_match.end():].strip()
                       self.in_json_block = False
                   else:
                       # no end marker found

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/ws_bom_robot_app/llm/providers/llm_manager.py

@@ -48,7 +48,7 @@ class OpenAI(LlmInterface):
             api_key=self.config.api_key or os.getenv("OPENAI_API_KEY"),
             model=self.config.model,
             stream_usage=True)
-        if not any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]):
+        if not (any(self.config.model.startswith(prefix) for prefix in ["o1", "o3"]) or "search" in self.config.model):
             chat.temperature = self.config.temperature
             chat.streaming = True
         return chat

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/ws_bom_robot_app/llm/tools/models/main.py

@@ -7,3 +7,7 @@ class DocumentRetrieverInput(BaseModel):
 class ImageGeneratorInput(BaseModel):
   query: str = Field(description="description of the image to generate.")
   language: str = Field(description="Language of the query. Default is 'it'", default="it")
+class LlmChainInput(BaseModel):
+  input: str = Field(description="Input to the LLM chain")
+class SearchOnlineInput(BaseModel):
+  query: str = Field(description="The search query string")

ws_bom_robot_app-0.0.59/ws_bom_robot_app/llm/tools/tool_manager.py

@@ -0,0 +1,188 @@
+from asyncio import Queue
+from typing import  Optional, Type, Callable
+from ws_bom_robot_app.llm.models.api import LlmAppTool
+from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
+from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
+from ws_bom_robot_app.llm.tools.utils import getRandomWaitingMessage, translate_text
+from ws_bom_robot_app.llm.tools.models.main import NoopInput,DocumentRetrieverInput,ImageGeneratorInput,LlmChainInput,SearchOnlineInput
+from pydantic import BaseModel, ConfigDict
+
+class ToolConfig(BaseModel):
+    function: Callable
+    model: Optional[Type[BaseModel]] = NoopInput
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True
+    )
+
+class ToolManager:
+    """
+    ToolManager is responsible for managing various tools used in the application.
+
+    Attributes:
+        app_tool (LlmAppTool): The application tool configuration.
+        api_key (str): The API key for accessing external services.
+        callbacks (list): A list of callback functions to be executed.
+
+    Methods:
+        document_retriever(query: str): Asynchronously retrieves documents based on the query.
+        image_generator(query: str, language: str = "it"): Asynchronously generates an image based on the query.
+        get_coroutine(): Retrieves the coroutine function based on the tool configuration.
+    """
+
+    def __init__(
+        self,
+        llm: LlmInterface,
+        app_tool: LlmAppTool,
+        callbacks: list,
+        queue: Optional[Queue] = None
+    ):
+        self.llm = llm
+        self.app_tool = app_tool
+        self.callbacks = callbacks
+        self.queue = queue
+
+    async def __extract_documents(self, query: str, app_tool: LlmAppTool):
+        search_type = "similarity"
+        search_kwargs = {"k": 4}
+        if app_tool.search_settings:
+            search_settings = app_tool.search_settings # type: ignore
+            if search_settings.search_type == "similarityScoreThreshold":
+                search_type = "similarity_score_threshold"
+                search_kwargs = {
+                    "score_threshold": search_settings.score_threshold_id if search_settings.score_threshold_id else  0.5,
+                    "k": search_settings.search_k if search_settings.search_k else 100
+                }
+            elif search_settings.search_type == "mmr":
+                search_type = "mmr"
+                search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
+            elif search_settings.search_type == "default":
+                search_type = "similarity"
+                search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
+            else:
+                search_type = "mixed"
+                search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
+        if self.queue:
+          await self.queue.put(getRandomWaitingMessage(app_tool.waiting_message, traduction=False))
+
+        return await VectorDbManager.get_strategy(app_tool.vector_type).invoke(
+            self.llm.get_embeddings(),
+            app_tool.vector_db,
+            query,
+            search_type,
+            search_kwargs,
+            app_tool=app_tool,
+            llm=self.llm.get_llm(),
+            source=app_tool.function_id,
+            )
+
+    #region functions
+    async def document_retriever(self, query: str) -> list:
+        """
+        Asynchronously retrieves documents based on the provided query using the specified search settings.
+
+        Args:
+          query (str): The search query string.
+
+        Returns:
+          list: A list of retrieved documents based on the search criteria.
+
+        Raises:
+          ValueError: If the configuration for the tool is invalid or the vector database is not found.
+
+        Notes:
+          - The function supports different search types such as "similarity", "similarity_score_threshold", "mmr", and "mixed".
+          - The search settings can be customized through the `app_tool.search_settings` attribute.
+          - If a queue is provided, a waiting message is put into the queue before invoking the search.
+        """
+        if (
+            self.app_tool.type == "function" and self.app_tool.vector_db
+            #and self.settings.get("dataSource") == "knowledgebase"
+        ):
+            return await self.__extract_documents(query, self.app_tool)
+
+    async def image_generator(self, query: str, language: str = "it"):
+        """
+        Asynchronously generates an image based on the query.
+        set OPENAI_API_KEY in your environment variables
+        """
+        from langchain_community.utilities.dalle_image_generator import DallEAPIWrapper
+        model = self.app_tool.model or "dall-e-3"
+        random_waiting_message = getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False)
+        if not language:
+            language = "it"
+        await translate_text(
+            self.llm, language, random_waiting_message, self.callbacks
+        )
+        try:
+            #set os.environ.get("OPENAI_API_KEY")!
+            image_url = DallEAPIWrapper(model=model).run(query)  # type: ignore
+            return image_url
+        except Exception as e:
+            return f"Error: {str(e)}"
+
+    async def llm_chain(self, input: str):
+        if self.app_tool.type == "llmChain":
+          from langchain_core.prompts import ChatPromptTemplate
+          from langchain_core.output_parsers import StrOutputParser
+          system_message = self.app_tool.llm_chain_settings.prompt
+          context = []
+          if self.app_tool.data_source == "knowledgebase":
+            context = await self.__extract_documents(input, self.app_tool)
+          if len(context) > 0:
+            for doc in context:
+              system_message += f"\n\nContext:\n{doc.metadata.get("source", "")}: {doc.page_content}"
+          prompt = ChatPromptTemplate.from_messages(
+              [ ("system", system_message),
+                ("user", "{input}")],
+          )
+          model = self.app_tool.llm_chain_settings.model
+          self.llm.config.model = model
+          llm = self.llm.get_llm()
+          chain = prompt | llm | StrOutputParser()
+          result = await chain.ainvoke({"input": input})
+          return result
+
+
+    async def search_online(self, query: str):
+        from ws_bom_robot_app.llm.tools.utils import fetch_page, extract_content_with_trafilatura
+        from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
+        import aiohttp, asyncio, ast
+        # Wrapper DuckDuckGo
+        search = DuckDuckGoSearchAPIWrapper(max_results=10)
+        try:
+          raw_results = search.results(query, max_results=10)
+        except Exception as e:
+            print(f"[!] Errore ricerca: {e}")
+        urls = [r["link"] for r in raw_results]
+        async with aiohttp.ClientSession() as session:
+            tasks = [fetch_page(session, url) for url in urls]
+            responses = await asyncio.gather(*tasks)
+        final_results = []
+        for item in responses:
+            url = item["url"]
+            html = item["html"]
+            if html:
+                content = await extract_content_with_trafilatura(html)
+                if content:
+                    final_results.append({"url": url, "content": content})
+                else:
+                    final_results.append({"url": url, "content": "No content found"})
+            else:
+                final_results.append({"url": url, "content": "Page not found"})
+        return final_results
+
+
+    #endregion
+
+    #class variables (static)
+    _list: dict[str,ToolConfig] = {
+        "document_retriever": ToolConfig(function=document_retriever, model=DocumentRetrieverInput),
+        "image_generator": ToolConfig(function=image_generator, model=ImageGeneratorInput),
+        "llm_chain": ToolConfig(function=llm_chain, model=LlmChainInput),
+        "search_online": ToolConfig(function=search_online, model=SearchOnlineInput),
+    }
+
+    #instance methods
+    def get_coroutine(self):
+        tool_cfg = self._list.get(self.app_tool.function_name)
+        return getattr(self, tool_cfg.function.__name__)  # type: ignore

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/ws_bom_robot_app/llm/tools/utils.py

@@ -23,3 +23,19 @@ async def translate_text(llm: LlmInterface, language, text: str, callbacks: list
   prompt = PromptTemplate.from_template(sys_message)
   chain = prompt | llm.get_llm()
   await chain.ainvoke({"language":language, "testo_da_tradurre": text}, {"callbacks": callbacks})
+
+async def fetch_page(session, url):
+    try:
+        async with session.get(url, timeout=10, ssl=False) as response:
+            if response.status == 200:
+                text = await response.text()
+                return {"url": url, "html": text}
+            else:
+                return {"url": url, "html": None}
+    except Exception as e:
+        return {"url": url, "html": None}
+
+async def extract_content_with_trafilatura(html):
+    """Estrae solo il testo principale usando trafilatura"""
+    import trafilatura
+    return trafilatura.extract(html)

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59/ws_bom_robot_app.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.57
+Version: 0.0.59
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa
@@ -17,15 +17,16 @@ Requires-Dist: pydantic==2.10.6
 Requires-Dist: pydantic-settings==2.7.1
 Requires-Dist: fastapi[standard]==0.115.8
 Requires-Dist: chevron==0.14.0
-Requires-Dist: langchain==0.3.18
-Requires-Dist: langchain-community==0.3.17
-Requires-Dist: langchain-core==0.3.34
-Requires-Dist: langchain-openai==0.3.5
+Requires-Dist: trafilatura==2.0.0
+Requires-Dist: langchain==0.3.25
+Requires-Dist: langchain-community==0.3.24
+Requires-Dist: langchain-core==0.3.59
+Requires-Dist: langchain-openai==0.3.16
 Requires-Dist: langchain-anthropic==0.3.6
 Requires-Dist: langchain-google-genai==2.0.7
 Requires-Dist: langchain-google-vertexai==2.0.13
-Requires-Dist: langchain-groq==0.2.4
-Requires-Dist: langchain-ollama==0.2.3
+Requires-Dist: langchain-groq==0.3.2
+Requires-Dist: langchain-ollama==0.3.2
 Requires-Dist: faiss-cpu==1.9.0
 Requires-Dist: chromadb==0.6.3
 Requires-Dist: langchain_chroma==0.2.1
@@ -48,6 +49,7 @@ Requires-Dist: unstructured-ingest[sharepoint]
 Requires-Dist: unstructured-ingest[slack]
 Requires-Dist: html5lib==1.1
 Requires-Dist: markdownify==0.14.1
+Requires-Dist: duckduckgo-search==8.0.4
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{ws_bom_robot_app-0.0.57 → ws_bom_robot_app-0.0.59}/ws_bom_robot_app.egg-info/requires.txt

@@ -5,15 +5,16 @@ pydantic==2.10.6
 pydantic-settings==2.7.1
 fastapi[standard]==0.115.8
 chevron==0.14.0
-langchain==0.3.18
-langchain-community==0.3.17
-langchain-core==0.3.34
-langchain-openai==0.3.5
+trafilatura==2.0.0
+langchain==0.3.25
+langchain-community==0.3.24
+langchain-core==0.3.59
+langchain-openai==0.3.16
 langchain-anthropic==0.3.6
 langchain-google-genai==2.0.7
 langchain-google-vertexai==2.0.13
-langchain-groq==0.2.4
-langchain-ollama==0.2.3
+langchain-groq==0.3.2
+langchain-ollama==0.3.2
 faiss-cpu==1.9.0
 chromadb==0.6.3
 langchain_chroma==0.2.1
@@ -36,3 +37,4 @@ unstructured-ingest[sharepoint]
 unstructured-ingest[slack]
 html5lib==1.1
 markdownify==0.14.1
+duckduckgo-search==8.0.4

ws_bom_robot_app-0.0.57/ws_bom_robot_app/llm/tools/tool_manager.py

@@ -1,134 +0,0 @@
-from asyncio import Queue
-from typing import  Optional, Type, Callable
-from ws_bom_robot_app.llm.models.api import LlmAppTool
-from ws_bom_robot_app.llm.providers.llm_manager import LlmInterface
-from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
-from ws_bom_robot_app.llm.tools.utils import getRandomWaitingMessage, translate_text
-from ws_bom_robot_app.llm.tools.models.main import NoopInput,DocumentRetrieverInput,ImageGeneratorInput
-from pydantic import BaseModel, ConfigDict
-
-class ToolConfig(BaseModel):
-    function: Callable
-    model: Optional[Type[BaseModel]] = NoopInput
-    model_config = ConfigDict(
-        arbitrary_types_allowed=True
-    )
-
-class ToolManager:
-    """
-    ToolManager is responsible for managing various tools used in the application.
-
-    Attributes:
-        app_tool (LlmAppTool): The application tool configuration.
-        api_key (str): The API key for accessing external services.
-        callbacks (list): A list of callback functions to be executed.
-
-    Methods:
-        document_retriever(query: str): Asynchronously retrieves documents based on the query.
-        image_generator(query: str, language: str = "it"): Asynchronously generates an image based on the query.
-        get_coroutine(): Retrieves the coroutine function based on the tool configuration.
-    """
-
-    def __init__(
-        self,
-        llm: LlmInterface,
-        app_tool: LlmAppTool,
-        callbacks: list,
-        queue: Optional[Queue] = None
-    ):
-        self.llm = llm
-        self.app_tool = app_tool
-        self.callbacks = callbacks
-        self.queue = queue
-
-
-    #region functions
-    async def document_retriever(self, query: str) -> list:
-        """
-        Asynchronously retrieves documents based on the provided query using the specified search settings.
-
-        Args:
-          query (str): The search query string.
-
-        Returns:
-          list: A list of retrieved documents based on the search criteria.
-
-        Raises:
-          ValueError: If the configuration for the tool is invalid or the vector database is not found.
-
-        Notes:
-          - The function supports different search types such as "similarity", "similarity_score_threshold", "mmr", and "mixed".
-          - The search settings can be customized through the `app_tool.search_settings` attribute.
-          - If a queue is provided, a waiting message is put into the queue before invoking the search.
-        """
-        if (
-            self.app_tool.type == "function" and self.app_tool.vector_db
-            #and self.settings.get("dataSource") == "knowledgebase"
-        ):
-            search_type = "similarity"
-            search_kwargs = {"k": 4}
-            if self.app_tool.search_settings:
-                search_settings = self.app_tool.search_settings # type: ignore
-                if search_settings.search_type == "similarityScoreThreshold":
-                    search_type = "similarity_score_threshold"
-                    search_kwargs = {
-                        "score_threshold": search_settings.score_threshold_id if search_settings.score_threshold_id else  0.5,
-                        "k": search_settings.search_k if search_settings.search_k else 100
-                    }
-                elif search_settings.search_type == "mmr":
-                    search_type = "mmr"
-                    search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
-                elif search_settings.search_type == "default":
-                    search_type = "similarity"
-                    search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
-                else:
-                    search_type = "mixed"
-                    search_kwargs = {"k": search_settings.search_k if search_settings.search_k else 4}
-            if self.queue:
-              await self.queue.put(getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False))
-
-            return await VectorDbManager.get_strategy(self.app_tool.vector_type).invoke(
-                self.llm.get_embeddings(),
-                self.app_tool.vector_db,
-                query,
-                search_type,
-                search_kwargs,
-                app_tool=self.app_tool,
-                llm=self.llm.get_llm(),
-                source=self.app_tool.function_id,
-                )
-        return []
-        #raise ValueError(f"Invalid configuration for {self.settings.name} tool of type {self.settings.type}. Must be a function or vector db not found.")
-
-    async def image_generator(self, query: str, language: str = "it"):
-        """
-        Asynchronously generates an image based on the query.
-        set OPENAI_API_KEY in your environment variables
-        """
-        from langchain_community.utilities.dalle_image_generator import DallEAPIWrapper
-        model = self.app_tool.model or "dall-e-3"
-        random_waiting_message = getRandomWaitingMessage(self.app_tool.waiting_message, traduction=False)
-        if not language:
-            language = "it"
-        await translate_text(
-            self.llm, language, random_waiting_message, self.callbacks
-        )
-        try:
-            #set os.environ.get("OPENAI_API_KEY")!
-            image_url = DallEAPIWrapper(model=model).run(query)  # type: ignore
-            return image_url
-        except Exception as e:
-            return f"Error: {str(e)}"
-
-    #endregion
-
-    #class variables (static)
-    _list: dict[str,ToolConfig] = {
-        "document_retriever": ToolConfig(function=document_retriever, model=DocumentRetrieverInput),
-        "image_generator": ToolConfig(function=image_generator, model=ImageGeneratorInput),
-    }
-
-    #instance methods
-    def get_coroutine(self):
-        tool_cfg = self._list.get(self.app_tool.function_name)
-        return getattr(self, tool_cfg.function.__name__)  # type: ignore