ws-bom-robot-app 0.0.105__py3-none-any.whl → 0.0.107__py3-none-any.whl

ws_bom_robot_app/llm/models/api.py

@@ -66,6 +66,7 @@ class LlmAppToolChainSettings(BaseModel):
 
 class LlmAppToolDbSettings(BaseModel):
   connection_string: Optional[str] = Field(None, validation_alias=AliasChoices("connectionString","connection_string"))
+  additionalPrompt: Optional[str] = Field(None, validation_alias=AliasChoices("additionalPrompt","additional_prompt"))
 
 class LlmAppTool(BaseModel):
   id: Optional[str] = None
@@ -235,6 +236,9 @@ class RulesRequest(VectorDbRequest):
   rules: List[str]
 
 class KbRequest(VectorDbRequest):
+  chucking_method: Optional[str] = Field("recursive", validation_alias=AliasChoices("chunkingMethod","chunking_method"))
+  chuck_size: Optional[int] = Field(3_000, validation_alias=AliasChoices("chunkSize","chuckt_size"))
+  chunk_overlap: Optional[int] = Field(300, validation_alias=AliasChoices("chunkOverlap","chunk_overlap"))
   files: Optional[List[str]] = []
   integrations: Optional[List[LlmKbIntegration]] = []
   endpoints: Optional[List[LlmKbEndpoint]] = []

ws_bom_robot_app/llm/tools/tool_manager.py

@@ -78,6 +78,80 @@ class ToolManager:
             source=app_tool.function_id,
             )
 
+    async def __download_sqlite_file(self, db_uri: str) -> str:
+        """
+        Scarica il file SQLite dalla CMS se necessario e restituisce il percorso locale.
+        Usa la stessa logica dell'integrazione Sitemap.
+
+        Args:
+            db_uri: URI del database o nome del file SQLite
+
+        Returns:
+            str: URI del database locale (sqlite:///path/to/file.db)
+        """
+        import os
+        from ws_bom_robot_app.config import config
+        from ws_bom_robot_app.llm.utils.download import download_file
+
+        if not db_uri.endswith('.db') and not db_uri.endswith('.sqlite') and not db_uri.endswith('.sqlite3'):
+            return db_uri
+
+        if db_uri.startswith('sqlite:///'):
+            file_path = db_uri.replace('sqlite:///', '')
+            if os.path.isabs(file_path) and os.path.exists(file_path):
+                return db_uri
+            filename = os.path.basename(file_path)
+        else:
+            filename = db_uri
+
+        db_folder = os.path.join(config.robot_data_folder, 'db')
+        os.makedirs(db_folder, exist_ok=True)
+
+        local_db_path = os.path.join(db_folder, filename)
+
+        if os.path.exists(local_db_path):
+            return f"sqlite:///{local_db_path}"
+
+        cms_file_url = f"{config.robot_cms_host}/{config.robot_cms_kb_folder}/{filename}"
+        auth = config.robot_cms_auth
+
+        try:
+            result = await download_file(cms_file_url, local_db_path, authorization=auth)
+            if result:
+                return f"sqlite:///{local_db_path}"
+            else:
+                raise ValueError(f"File SQLite {filename} non trovato nella CMS")
+        except Exception as e:
+            raise ValueError(f"Errore durante il download del file SQLite {filename}: {str(e)}")
+
+    async def __query_database(self, query: str, app_tool: LlmAppTool):
+        from langchain_community.agent_toolkits.sql.base import create_sql_agent
+        from langchain_community.utilities import SQLDatabase
+
+        secrets = app_tool.secrets_to_dict()
+
+        db_uri = app_tool.db_settings.connection_string
+        additional_prompt = app_tool.db_settings.additionalPrompt
+        if not db_uri:
+            raise ValueError("Database URI not found in tool secrets")
+
+        db_uri = await self.__download_sqlite_file(db_uri)
+
+        db = SQLDatabase.from_uri(db_uri)
+        llm = self.llm.get_llm()
+
+        agent = create_sql_agent(
+            llm=llm,
+            db=db,
+            agent_type="tool-calling",
+            suffix=additional_prompt if additional_prompt else None,
+        )
+
+        result = await agent.ainvoke({"input": query}, config={"callbacks": []})
+        if result and "output" in result:
+            return result["output"]
+        return None
+
     #region functions
     async def document_retriever(self, query: str) -> list:
         """
@@ -99,9 +173,11 @@ class ToolManager:
         """
         if (
             self.app_tool.type == "function" and self.app_tool.vector_db
-            #and self.settings.get("dataSource") == "knowledgebase"
+            and self.app_tool.data_source == "knowledgebase"
         ):
             return await self.__extract_documents(query, self.app_tool)
+        elif self.app_tool.type == "function" and self.app_tool.data_source == "database":
+            return await self.__query_database(query, self.app_tool)
 
     async def image_generator(self, query: str, language: str = "it"):
         """

ws_bom_robot_app/llm/utils/chunker.py

@@ -1,13 +1,17 @@
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_text_splitters import MarkdownHeaderTextSplitter
 
 class DocumentChunker:
   @staticmethod
-  def chunk(documents: list[Document]) -> list[Document]:
-       return DocumentChunker.chunk_recursive(documents)
+  def chunk(documents: list[Document], chucking_method: str = "recursive", chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+      if chucking_method == "recursive":
+        return DocumentChunker.chunk_recursive(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+      elif chucking_method == "markdownHeader":
+        return DocumentChunker.chunk_markdown(documents, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
 
   @staticmethod
-  def chunk_recursive(documents: list[Document], chunk_size: int=3_000) -> list[Document]:
+  def chunk_recursive(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
         """
         Recursively split documents into smaller chunks while preserving metadata.
 
@@ -17,16 +21,16 @@ class DocumentChunker:
 
         Args:
             documents (list[Document]): A list of Document objects to be chunked.
-            chunk_size (int, optional): The maximum size of each chunk in characters. 
+            chunk_size (int, optional): The maximum size of each chunk in characters.
                 Defaults to 3,000.
 
         Returns:
-            list[Document]: A list of Document objects where each document's content is 
-                at most chunk_size characters. Each chunk preserves the metadata from 
+            list[Document]: A list of Document objects where each document's content is
+                at most chunk_size characters. Each chunk preserves the metadata from
                 its original document.
 
         Notes:
-            - Chunk overlap is automatically set to 10% of the chunk_size to maintain 
+            - Chunk overlap is automatically set to 10% of the chunk_size to maintain
               context between chunks.
             - Documents smaller than or equal to chunk_size are returned unchanged.
             - Metadata from the original document is copied to all resulting chunks.
@@ -80,3 +84,69 @@ class DocumentChunker:
                 )
         return chunked_documents
 
+  @staticmethod
+  def chunk_markdown(documents: list[Document], chunk_size: int=3_000, chunk_overlap: int=300) -> list[Document]:
+        """
+        Splits markdown documents based on headers and then into smaller chunks.
+
+        This function takes a list of Document objects containing markdown content and splits
+        them based on markdown headers (# H1, ## H2, ### H3, etc.). After splitting by headers,
+        it further chunks large sections using RecursiveCharacterTextSplitter to ensure no
+        chunk exceeds the specified size.
+
+        Args:
+            documents (list[Document]): A list of Document objects with markdown content.
+            chunk_size (int, optional): The maximum size of each chunk in characters.
+                Defaults to 3,000.
+
+        Returns:
+            list[Document]: A list of Document objects where each document represents a chunk.
+                Metadata includes the markdown header hierarchy and original document metadata.
+
+        Note:
+            - Headers are split at levels: H1 (#), H2 (##), and H3 (###)
+            - Header information is preserved in the metadata
+            - Large sections are further split to respect chunk_size limit
+            - Chunk overlap is set to 10% of chunk_size for context preservation
+        """
+        # Define headers to split on
+        headers_to_split_on = [
+            ("#", "h1"),
+            ("##", "h2"),
+            ("###", "h3"),
+            ("####", "h4"),
+        ]
+
+        markdown_splitter = MarkdownHeaderTextSplitter(
+            headers_to_split_on=headers_to_split_on,
+            strip_headers=False
+        )
+
+        # Secondary splitter for large sections
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+
+        chunked_documents = []
+        for doc in documents:
+            # First split by markdown headers
+            md_chunks = markdown_splitter.split_text(doc.page_content)
+
+            # Then split large sections if needed
+            for md_chunk in md_chunks:
+                # Merge metadata from original doc and header metadata
+                merged_metadata = {**doc.metadata, **md_chunk.metadata}
+                if len(md_chunk.page_content) <= chunk_size:
+                    chunked_documents.append(
+                        Document(page_content=md_chunk.page_content, metadata=merged_metadata)
+                    )
+                else:
+                    # Further split large sections
+                    sub_chunks = text_splitter.split_text(md_chunk.page_content)
+                    for sub_chunk in sub_chunks:
+                        chunked_documents.append(
+                            Document(page_content=sub_chunk, metadata=merged_metadata)
+                        )
+
+        return chunked_documents

ws_bom_robot_app/llm/vector_store/db/base.py

@@ -107,6 +107,9 @@ class VectorDBStrategy(ABC):
         embeddings: Embeddings,
         documents: List[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         pass

ws_bom_robot_app/llm/vector_store/db/chroma.py

@@ -46,11 +46,14 @@ class Chroma(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: CHROMA = None

ws_bom_robot_app/llm/vector_store/db/faiss.py

@@ -30,11 +30,14 @@ class Faiss(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
             _instance: FAISS = None

ws_bom_robot_app/llm/vector_store/db/qdrant.py

@@ -14,14 +14,17 @@ class Qdrant(VectorDBStrategy):
         embeddings: Embeddings,
         documents: list[Document],
         storage_id: str,
+        chucking_method: str,
+        chunk_size: int,
+        chunk_overlap: int,
         **kwargs
     ) -> Optional[str]:
         try:
             documents = self._remove_empty_documents(documents)
-            chunked_docs = DocumentChunker.chunk(documents)
+            chunked_docs = DocumentChunker.chunk(documents, chucking_method, chunk_size, chunk_overlap)
             batches = self._batch_documents_by_tokens(chunked_docs)
             logging.info(f"documents: {len(documents)}, after chunking: {len(chunked_docs)}, processing batches: {len(batches)}")
-            _instance: QDRANT = None            
+            _instance: QDRANT = None
             if not os.path.exists(storage_id):
                 os.makedirs(storage_id)
 
@@ -45,10 +48,10 @@ class Qdrant(VectorDBStrategy):
                 # add a small delay to avoid rate limiting
                 if i < len(batches) - 1:  # except last batch
                     await asyncio.sleep(1)
-            if _instance:                
+            if _instance:
                 self._clear_cache(storage_id)
                 logging.info(f"Successfully created {Qdrant.__name__} index with {len(chunked_docs)} total documents")
-            return storage_id                
+            return storage_id
         except Exception as e:
             logging.error(f"{Qdrant.__name__} create error: {e}")
             raise e

ws_bom_robot_app/llm/vector_store/generator.py

@@ -112,7 +112,7 @@ async def kb(rq: KbRequest) -> VectorDbResponse:
       db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
           os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
           "zip",
-          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
+          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, rq.chucking_method, rq.chuck_size, rq.chunk_overlap, return_folder_path=True)
       )
       return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
     except Exception as e:

ws_bom_robot_app/llm/vector_store/integration/api.py

@@ -1,216 +1,216 @@
-import asyncio, logging, aiohttp
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import List, Union, Optional, Dict, Any, Literal
-from pydantic import BaseModel, Field, AliasChoices, field_validator
-import json
-import os
-
-
-class AuthConfig(BaseModel):
-  """
-  Configuration for API authentication.
-
-  Attributes:
-    type: Type of authentication (bearer, basic, api_key, custom, none)
-    token: Bearer token or API key value
-    username: Username for basic auth
-    password: Password for basic auth
-    header_name: Custom header name for API key
-    prefix: Prefix for the auth value (e.g., 'Bearer', 'Token')
-  """
-  type: Literal["bearer", "basic", "api_key", "custom", "none"] = Field(default="none")
-  token: Optional[str] = Field(default=None)
-  username: Optional[str] = Field(default=None)
-  password: Optional[str] = Field(default=None)
-  header_name: Optional[str] = Field(default=None, validation_alias=AliasChoices("headerName", "header_name"))
-  prefix: Optional[str] = Field(default=None)
-
-
-class ApiParams(BaseModel):
-  """
-  Generic API Integration Parameters.
-
-  Attributes:
-    url: The base URL of the API endpoint
-    method: HTTP method (GET, POST, PUT, DELETE, PATCH)
-    headers: Custom headers to include in the request
-    params: Query parameters for the request
-    body: Request body for POST/PUT/PATCH requests
-    auth: Authentication configuration
-    response_data_path: JSON path to extract data from response (e.g., 'data.items', 'results')
-    max_retries: Maximum number of retry attempts for failed requests
-    retry_delay: Base delay in seconds between retries (uses exponential backoff)
-    timeout: Request timeout in seconds
-  """
-  url: str = Field(validation_alias=AliasChoices("url", "endpoint"))
-  method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"] = Field(default="GET")
-  headers: Optional[Dict[str, str]] = Field(default_factory=dict)
-  params: Optional[Dict[str, Any]] = Field(default_factory=dict)
-  body: Optional[Union[Dict[str, Any], str]] = Field(default=None)
-  auth: Optional[AuthConfig] = Field(default_factory=lambda: AuthConfig())
-  response_data_path: Optional[str] = Field(default=None, validation_alias=AliasChoices("responseDataPath", "response_data_path"))
-  max_retries: int = Field(default=5, validation_alias=AliasChoices("maxRetries", "max_retries"))
-  retry_delay: float = Field(default=1.0, validation_alias=AliasChoices("retryDelay", "retry_delay"))
-  timeout: int = Field(default=30)
-
-  @field_validator('auth', mode='before')
-  @classmethod
-  def parse_auth(cls, v):
-    """Parse auth config from dict if needed"""
-    if isinstance(v, dict):
-      return AuthConfig(**v)
-    return v or AuthConfig()
-
-
-class Api(IntegrationStrategy):
-  """
-  Generic API Integration that supports:
-  - Multiple HTTP methods (GET, POST, PUT, DELETE, PATCH)
-  - Various authentication types (Bearer, Basic, API Key, Custom)
-  - Custom headers and parameters
-  - Automatic retry with exponential backoff
-  - Flexible response data extraction
-  """
-
-  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str, int, list]]):
-    super().__init__(knowledgebase_path, data)
-    self.__data = ApiParams.model_validate(self.data)
-
-  def working_subdirectory(self) -> str:
-    return 'api_integration'
-
-  async def run(self) -> None:
-    """Fetch data from the API and save to JSON file"""
-    _data = await self.__fetch_data()
-    json_file_path = os.path.join(self.working_directory, 'api_data.json')
-    with open(json_file_path, 'w', encoding='utf-8') as f:
-      json.dump(_data, f, ensure_ascii=False, indent=2)
-    logging.info(f"Saved {len(_data) if isinstance(_data, list) else 1} items to {json_file_path}")
-
-  async def load(self) -> list[Document]:
-    """Load data from API and convert to documents"""
-    await self.run()
-    await asyncio.sleep(1)
-    return await Loader(self.working_directory).load()
-
-  def __prepare_headers(self) -> Dict[str, str]:
-    """Prepare request headers with authentication"""
-    headers = self.__data.headers.copy() if self.__data.headers else {}
-
-    # Add Content-Type if not present
-    if 'Content-Type' not in headers and self.__data.method in ["POST", "PUT", "PATCH"]:
-      headers['Content-Type'] = 'application/json'
-
-    # Add authentication
-    auth = self.__data.auth
-    if auth.type == "bearer":
-      prefix = auth.prefix or "Bearer"
-      headers['Authorization'] = f"{prefix} {auth.token}"
-    elif auth.type == "basic":
-      import base64
-      credentials = f"{auth.username}:{auth.password}"
-      encoded = base64.b64encode(credentials.encode()).decode()
-      headers['Authorization'] = f"Basic {encoded}"
-    elif auth.type == "api_key" and auth.header_name:
-      prefix = f"{auth.prefix} " if auth.prefix else ""
-      headers[auth.header_name] = f"{prefix}{auth.token}"
-
-    return headers
-
-  def __get_nested_value(self, data: Any, path: Optional[str]) -> Any:
-    """Extract nested value from data using dot notation path"""
-    if not path:
-      return data
-
-    keys = path.split('.')
-    current = data
-    for key in keys:
-      if isinstance(current, dict):
-        current = current.get(key)
-      elif isinstance(current, list) and key.isdigit():
-        current = current[int(key)]
-      else:
-        return None
-
-      if current is None:
-        return None
-
-    return current
-
-  async def __make_request(
-    self,
-    url: str,
-    headers: Dict[str, str],
-    params: Optional[Dict[str, Any]] = None
-  ) -> Dict[str, Any]:
-    """Make HTTP request with retry logic"""
-    retry_count = 0
-
-    while retry_count <= self.__data.max_retries:
-      try:
-        timeout = aiohttp.ClientTimeout(total=self.__data.timeout)
-
-        async with aiohttp.ClientSession(timeout=timeout) as session:
-          request_kwargs = {
-            "headers": headers,
-            "params": params or self.__data.params
-          }
-
-          # Add body for POST/PUT/PATCH
-          if self.__data.method in ["POST", "PUT", "PATCH"] and self.__data.body:
-            if isinstance(self.__data.body, dict):
-              request_kwargs["json"] = self.__data.body
-            else:
-              request_kwargs["data"] = self.__data.body
-
-          async with session.request(
-            self.__data.method,
-            url,
-            **request_kwargs
-          ) as response:
-            # Check response status
-            if response.status == 429:  # Rate limit
-              retry_count += 1
-              if retry_count > self.__data.max_retries:
-                raise Exception("Rate limit exceeded. Maximum retries reached.")
-
-              wait_time = self.__data.retry_delay * (2 ** retry_count)
-              logging.warning(f"Rate limited. Waiting {wait_time}s (Attempt {retry_count}/{self.__data.max_retries})")
-              await asyncio.sleep(wait_time)
-              continue
-
-            response.raise_for_status()
-
-            # Parse response
-            try:
-              data = await response.json()
-              return data
-            except aiohttp.ContentTypeError:
-              text = await response.text()
-              logging.warning(f"Non-JSON response received: {text[:200]}")
-              return {"text": text}
-
-      except aiohttp.ClientError as e:
-        retry_count += 1
-        if retry_count > self.__data.max_retries:
-          raise Exception(f"Request failed after {self.__data.max_retries} retries: {e}")
-
-        wait_time = self.__data.retry_delay * (2 ** retry_count)
-        logging.warning(f"Request error: {e}. Retrying in {wait_time}s...")
-        await asyncio.sleep(wait_time)
-        continue
-
-    raise Exception("Maximum retries exceeded")
-
-  async def __fetch_data(self) -> Any:
-    """Fetch data from API"""
-    headers = self.__prepare_headers()
-    response = await self.__make_request(self.__data.url, headers)
-
-    # Extract data from response using path if specified
-    data = self.__get_nested_value(response, self.__data.response_data_path)
-    result = data if data is not None else response
-
-    return result
+import asyncio, logging, aiohttp
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import List, Union, Optional, Dict, Any, Literal
+from pydantic import BaseModel, Field, AliasChoices, field_validator
+import json
+import os
+
+
+class AuthConfig(BaseModel):
+  """
+  Configuration for API authentication.
+
+  Attributes:
+    type: Type of authentication (bearer, basic, api_key, custom, none)
+    token: Bearer token or API key value
+    username: Username for basic auth
+    password: Password for basic auth
+    header_name: Custom header name for API key
+    prefix: Prefix for the auth value (e.g., 'Bearer', 'Token')
+  """
+  type: Literal["bearer", "basic", "api_key", "custom", "none"] = Field(default="none")
+  token: Optional[str] = Field(default=None)
+  username: Optional[str] = Field(default=None)
+  password: Optional[str] = Field(default=None)
+  header_name: Optional[str] = Field(default=None, validation_alias=AliasChoices("headerName", "header_name"))
+  prefix: Optional[str] = Field(default=None)
+
+
+class ApiParams(BaseModel):
+  """
+  Generic API Integration Parameters.
+
+  Attributes:
+    url: The base URL of the API endpoint
+    method: HTTP method (GET, POST, PUT, DELETE, PATCH)
+    headers: Custom headers to include in the request
+    params: Query parameters for the request
+    body: Request body for POST/PUT/PATCH requests
+    auth: Authentication configuration
+    response_data_path: JSON path to extract data from response (e.g., 'data.items', 'results')
+    max_retries: Maximum number of retry attempts for failed requests
+    retry_delay: Base delay in seconds between retries (uses exponential backoff)
+    timeout: Request timeout in seconds
+  """
+  url: str = Field(validation_alias=AliasChoices("url", "endpoint"))
+  method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"] = Field(default="GET")
+  headers: Optional[Dict[str, str]] = Field(default_factory=dict)
+  params: Optional[Dict[str, Any]] = Field(default_factory=dict)
+  body: Optional[Union[Dict[str, Any], str]] = Field(default=None)
+  auth: Optional[AuthConfig] = Field(default_factory=lambda: AuthConfig())
+  response_data_path: Optional[str] = Field(default=None, validation_alias=AliasChoices("responseDataPath", "response_data_path"))
+  max_retries: int = Field(default=5, validation_alias=AliasChoices("maxRetries", "max_retries"))
+  retry_delay: float = Field(default=1.0, validation_alias=AliasChoices("retryDelay", "retry_delay"))
+  timeout: int = Field(default=30)
+
+  @field_validator('auth', mode='before')
+  @classmethod
+  def parse_auth(cls, v):
+    """Parse auth config from dict if needed"""
+    if isinstance(v, dict):
+      return AuthConfig(**v)
+    return v or AuthConfig()
+
+
+class Api(IntegrationStrategy):
+  """
+  Generic API Integration that supports:
+  - Multiple HTTP methods (GET, POST, PUT, DELETE, PATCH)
+  - Various authentication types (Bearer, Basic, API Key, Custom)
+  - Custom headers and parameters
+  - Automatic retry with exponential backoff
+  - Flexible response data extraction
+  """
+
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str, int, list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = ApiParams.model_validate(self.data)
+
+  def working_subdirectory(self) -> str:
+    return 'api_integration'
+
+  async def run(self) -> None:
+    """Fetch data from the API and save to JSON file"""
+    _data = await self.__fetch_data()
+    json_file_path = os.path.join(self.working_directory, 'api_data.json')
+    with open(json_file_path, 'w', encoding='utf-8') as f:
+      json.dump(_data, f, ensure_ascii=False, indent=2)
+    logging.info(f"Saved {len(_data) if isinstance(_data, list) else 1} items to {json_file_path}")
+
+  async def load(self) -> list[Document]:
+    """Load data from API and convert to documents"""
+    await self.run()
+    await asyncio.sleep(1)
+    return await Loader(self.working_directory).load()
+
+  def __prepare_headers(self) -> Dict[str, str]:
+    """Prepare request headers with authentication"""
+    headers = self.__data.headers.copy() if self.__data.headers else {}
+
+    # Add Content-Type if not present
+    if 'Content-Type' not in headers and self.__data.method in ["POST", "PUT", "PATCH"]:
+      headers['Content-Type'] = 'application/json'
+
+    # Add authentication
+    auth = self.__data.auth
+    if auth.type == "bearer":
+      prefix = auth.prefix or "Bearer"
+      headers['Authorization'] = f"{prefix} {auth.token}"
+    elif auth.type == "basic":
+      import base64
+      credentials = f"{auth.username}:{auth.password}"
+      encoded = base64.b64encode(credentials.encode()).decode()
+      headers['Authorization'] = f"Basic {encoded}"
+    elif auth.type == "api_key" and auth.header_name:
+      prefix = f"{auth.prefix} " if auth.prefix else ""
+      headers[auth.header_name] = f"{prefix}{auth.token}"
+
+    return headers
+
+  def __get_nested_value(self, data: Any, path: Optional[str]) -> Any:
+    """Extract nested value from data using dot notation path"""
+    if not path:
+      return data
+
+    keys = path.split('.')
+    current = data
+    for key in keys:
+      if isinstance(current, dict):
+        current = current.get(key)
+      elif isinstance(current, list) and key.isdigit():
+        current = current[int(key)]
+      else:
+        return None
+
+      if current is None:
+        return None
+
+    return current
+
+  async def __make_request(
+    self,
+    url: str,
+    headers: Dict[str, str],
+    params: Optional[Dict[str, Any]] = None
+  ) -> Dict[str, Any]:
+    """Make HTTP request with retry logic"""
+    retry_count = 0
+
+    while retry_count <= self.__data.max_retries:
+      try:
+        timeout = aiohttp.ClientTimeout(total=self.__data.timeout)
+
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+          request_kwargs = {
+            "headers": headers,
+            "params": params or self.__data.params
+          }
+
+          # Add body for POST/PUT/PATCH
+          if self.__data.method in ["POST", "PUT", "PATCH"] and self.__data.body:
+            if isinstance(self.__data.body, dict):
+              request_kwargs["json"] = self.__data.body
+            else:
+              request_kwargs["data"] = self.__data.body
+
+          async with session.request(
+            self.__data.method,
+            url,
+            **request_kwargs
+          ) as response:
+            # Check response status
+            if response.status == 429:  # Rate limit
+              retry_count += 1
+              if retry_count > self.__data.max_retries:
+                raise Exception("Rate limit exceeded. Maximum retries reached.")
+
+              wait_time = self.__data.retry_delay * (2 ** retry_count)
+              logging.warning(f"Rate limited. Waiting {wait_time}s (Attempt {retry_count}/{self.__data.max_retries})")
+              await asyncio.sleep(wait_time)
+              continue
+
+            response.raise_for_status()
+
+            # Parse response
+            try:
+              data = await response.json()
+              return data
+            except aiohttp.ContentTypeError:
+              text = await response.text()
+              logging.warning(f"Non-JSON response received: {text[:200]}")
+              return {"text": text}
+
+      except aiohttp.ClientError as e:
+        retry_count += 1
+        if retry_count > self.__data.max_retries:
+          raise Exception(f"Request failed after {self.__data.max_retries} retries: {e}")
+
+        wait_time = self.__data.retry_delay * (2 ** retry_count)
+        logging.warning(f"Request error: {e}. Retrying in {wait_time}s...")
+        await asyncio.sleep(wait_time)
+        continue
+
+    raise Exception("Maximum retries exceeded")
+
+  async def __fetch_data(self) -> Any:
+    """Fetch data from API"""
+    headers = self.__prepare_headers()
+    response = await self.__make_request(self.__data.url, headers)
+
+    # Extract data from response using path if specified
+    data = self.__get_nested_value(response, self.__data.response_data_path)
+    result = data if data is not None else response
+
+    return result

{ws_bom_robot_app-0.0.105.dist-info → ws_bom_robot_app-0.0.107.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.105
+Version: 0.0.107
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.105.dist-info → ws_bom_robot_app-0.0.107.dist-info}/RECORD

@@ -19,7 +19,7 @@ ws_bom_robot_app/llm/nebuly_handler.py,sha256=wFO2UG849kv5hmjM5EoOp0Jsloy-BtQjrR
 ws_bom_robot_app/llm/feedbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/feedbacks/feedback_manager.py,sha256=vNcZLG9IKhurAk7hjBqyFgQTjnh3Cd4GnxeYsX7ZdiA,2922
 ws_bom_robot_app/llm/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/models/api.py,sha256=QfJ--9ONGi0b6Ua5km7N2MM7fkT1lu2kSRxMBg6ZlbI,12346
+ws_bom_robot_app/llm/models/api.py,sha256=jvoU8z82L7xGkqo2a2m--0OxZrENoLVPuucA-rdr74o,12798
 ws_bom_robot_app/llm/models/base.py,sha256=1TqxuTK3rjJEALn7lvgoen_1ba3R2brAgGx6EDTtDZo,152
 ws_bom_robot_app/llm/models/feedback.py,sha256=pYNQGxNOBgeAAfdJLI95l7ePLBI5tVdsgnyjp5oMOQU,1722
 ws_bom_robot_app/llm/models/kb.py,sha256=oVSw6_dmNxikAHrPqcfxDXz9M0ezLIYuxpgvzfs_Now,9514
@@ -27,13 +27,13 @@ ws_bom_robot_app/llm/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 ws_bom_robot_app/llm/providers/llm_manager.py,sha256=oVeEmZUnR1ysV-BI_zpwQ-gpXqmhSzjKFQQAHtaFGSI,16596
 ws_bom_robot_app/llm/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/tools/tool_builder.py,sha256=CtZwJ94aj0YGA3yVWkyCUxNE7WgU2zWjhl_tEfEskxw,3432
-ws_bom_robot_app/llm/tools/tool_manager.py,sha256=ThiaBCDOn192NOaLHsxbNZXYs5fptqlfgCHW-9h2eVY,15989
+ws_bom_robot_app/llm/tools/tool_manager.py,sha256=6qPzFbcp9zH3tiNMfGBPV_N4WAp65LUIfmjOXvNU8mI,18973
 ws_bom_robot_app/llm/tools/utils.py,sha256=Ba7ScFZPVJ3ke8KLO8ik1wyR2f_zC99Bikqx0OGnKoI,1924
 ws_bom_robot_app/llm/tools/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/tools/models/main.py,sha256=1hICqHs-KS2heenkH7b2eH0N2GrPaaNGBrn64cl_A40,827
 ws_bom_robot_app/llm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/utils/agent.py,sha256=uFuSfYMfGIE2WCKGNSKL-T2SDFn-tUKvbAYbGTPIw6g,1445
-ws_bom_robot_app/llm/utils/chunker.py,sha256=u0l2t3bIQihOpLRlcrO23bNBda3kgzayyGAIR0YZUqQ,4069
+ws_bom_robot_app/llm/utils/chunker.py,sha256=-WfDG6xUU_oOUJmWhDlQbI1hsGCRkmnKyqkY_bEG8WA,7420
 ws_bom_robot_app/llm/utils/cleanup.py,sha256=ARLZTX4mLbkLCEnMdIWYDYEAPOjzfy1laLGkYnxZe30,3063
 ws_bom_robot_app/llm/utils/cms.py,sha256=gfIXvY3DxgbgDf0LCzyekWitaduxKGLHfV6gbRmh8zk,6960
 ws_bom_robot_app/llm/utils/download.py,sha256=rvc88E63UGHnFVlJJeMb05Z2FcBYIITqKnIE3ldEu6I,7293
@@ -41,15 +41,15 @@ ws_bom_robot_app/llm/utils/print.py,sha256=HK3zhZOd4cEyXZ8QcudLtTIfqqtMOERce_yTo
 ws_bom_robot_app/llm/utils/secrets.py,sha256=-HtqLIDVIJrpvGC5YhPAVyLsq8P4ChVM5g3GOfdwqVk,878
 ws_bom_robot_app/llm/utils/webhooks.py,sha256=LAAZqyN6VhV13wu4X-X85TwdDgAV2rNvIwQFIIc0FJM,2114
 ws_bom_robot_app/llm/vector_store/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/generator.py,sha256=W_hi_UOPaSjnEuazhUFIrMAwTvz64Du8_gpiVAxFlVc,6451
+ws_bom_robot_app/llm/vector_store/generator.py,sha256=nSj8aLARr4h1SJlkEI7X1hDef195fAPKEi2fFkl7_wM,6504
 ws_bom_robot_app/llm/vector_store/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/db/base.py,sha256=pIlHTg83bLdGfbZorilSqeJ5QKgpGU8fxF3c-5pLWJo,8490
-ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=oe0p_OlqTGFXEvtC8AXiElUWnfHXTXk97_suYh7kirU,4622
-ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=rCMq_dhg1-NM8G5L_VEdDIvgmkWLXL3r5EreaqxR3Oc,3925
+ws_bom_robot_app/llm/vector_store/db/base.py,sha256=GhTkOq4ms_vUf_nuncyskUpI6kWPKDQi5dfLU5zduFY,8576
+ws_bom_robot_app/llm/vector_store/db/chroma.py,sha256=9tnEKQLvBt5TPthULR08ktDkcpFjuIxuYV7REFp9kuY,4752
+ws_bom_robot_app/llm/vector_store/db/faiss.py,sha256=lHpBZV1s_OZTiRlcVM-KJBf2wWWkzvYm_gt57BdbbUs,4055
 ws_bom_robot_app/llm/vector_store/db/manager.py,sha256=5rqBvc0QKmHFUgVHqBAr1Y4FZRl-w-ylGMjgXZywrdA,533
-ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=-36YOXjNtDeWveREnGd1SZF3hT7_Peg_pAT5uoxXcQU,3237
+ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=v3YKLZ9_ysaNB64UVA1JCYg-W1BMGfo9CLCG4roXtJ4,3323
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ws_bom_robot_app/llm/vector_store/integration/api.py,sha256=0fW2uKW3P_WaX8J18LRTMTVZ1VP3t_bHkocDtm1f_uc,8184
+ws_bom_robot_app/llm/vector_store/integration/api.py,sha256=jivsqw3iMr4isnxi-jQYtFWPtBcTgIDe88hiUqXv5NE,8400
 ws_bom_robot_app/llm/vector_store/integration/azure.py,sha256=OEa96Dlf1CX0tjrTjX4KP3D_HTn249ukc9sluPbdOyU,3389
 ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=Aat4vQCmsrTiWrzUi2_h-RKzJLhObsKFfZrzs1TnQP8,4385
 ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=TMmGe53tHRTgHJ7nA8DqZVodo3aMEzHrrSdl0-I0-S0,4350
@@ -70,7 +70,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=InpRwKPxp0tuM4drezBvxxAWHe3XTmu60MGvFsT7RPE,7176
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=RFYSZkZAYtU8wJSd1rN2T0lVo-wK1-ddtr6bH2fBr6Q,5170
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.105.dist-info/METADATA,sha256=dOAkYLVKxkr3-FcVVFK475-jH80yd_s3DQSB4bLnl0c,11011
-ws_bom_robot_app-0.0.105.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.105.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.105.dist-info/RECORD,,
+ws_bom_robot_app-0.0.107.dist-info/METADATA,sha256=ctg-rgQKDvVvLr_L_wJxP1FwhNdl6NZRYdYB5JhV9Ys,11011
+ws_bom_robot_app-0.0.107.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+ws_bom_robot_app-0.0.107.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.107.dist-info/RECORD,,

{ws_bom_robot_app-0.0.105.dist-info → ws_bom_robot_app-0.0.107.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any