ws-bom-robot-app 0.0.87__py3-none-any.whl → 0.0.89__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/generator.py

@@ -1,137 +1,137 @@
-import os, gc, shutil, logging, traceback
-import asyncio, aiofiles, aiofiles.os
-from fastapi import HTTPException
-from fastapi.responses import StreamingResponse
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
-from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
-from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
-from ws_bom_robot_app.config import config
-from ws_bom_robot_app.llm.models.kb import load_endpoints
-from ws_bom_robot_app.llm.utils.download import download_files
-
-async def _cleanup_directory(directory_path: str):
-  if os.path.exists(directory_path):
-    await asyncio.to_thread(shutil.rmtree, directory_path)
-
-#@timer
-async def rules(rq: RulesRequest) -> VectorDbResponse:
-  _config = rq.config()
-  db_name = rq.out_name()
-  store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
-  try:
-    await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
-    db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
-    return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
-  except Exception as e:
-    try:
-      await _cleanup_directory(store_path)
-    finally:
-      return VectorDbResponse(success = False, error = str(e))
-  finally:
-    gc.collect()
-
-#@atimer
-async def kb(rq: KbRequest) -> VectorDbResponse:
-  os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
-  _config = rq.config()
-  db_name = rq.out_name()
-  src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
-  working_path = os.path.join(src_path, db_name)
-
-  if all([not rq.files,not rq.endpoints,not rq.integrations]):
-    return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
-  else:
-    await aiofiles.os.makedirs(src_path, exist_ok=True)
-    await aiofiles.os.makedirs(working_path, exist_ok=True)
-
-  documents: list[Document] = []
-  # Download/copy all files
-  if rq.files:
-    try:
-      loaders = Loader(working_path)
-      filter_file_extensions = loaders.managed_file_extensions()
-      files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
-      if files_to_download:
-        await download_files(
-          [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
-          src_path, authorization=_config.robot_cms_auth)
-      # copy files to working tmp folder
-      for file in rq.files:
-        async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
-          async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
-            await dest_file.write(await src_file.read())
-      #load files
-      try:
-        documents.extend(await loaders.load())
-      except Exception as e:
-        tb = traceback.format_exc()
-        _error = f"File loader failure: {e} | {tb}"
-        logging.warning(_error)
-        return VectorDbResponse(success = False, error = _error)
-    except Exception as e:
-      await _cleanup_directory(working_path)
-      return VectorDbResponse(success = False, error = f"Failed to download file {e}")
-
-  if rq.endpoints:
-    try:
-      documents.extend(await load_endpoints(rq.endpoints, working_path))
-    except Exception as e:
-      await _cleanup_directory(working_path)
-      tb = traceback.format_exc()
-      _error = f"Endpoint failure: {e} | {tb}"
-      logging.warning(_error)
-      return VectorDbResponse(success = False, error = _error)
-
-  if rq.integrations:
-    tasks = []
-    for integration in rq.integrations:
-      tasks.append(
-        IntegrationManager
-        .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
-        .load()
-      )
-    try:
-      integration_documents = await asyncio.gather(*tasks)
-      for docs in integration_documents:
-        documents.extend(docs)
-    except Exception as e:
-      await _cleanup_directory(working_path)
-      tb = traceback.format_exc()
-      _error = f"Integration failure: {e} | {tb}"
-      logging.warning(_error)
-      return VectorDbResponse(success=False, error=_error)
-
-  #cleanup
-  await _cleanup_directory(working_path)
-
-  if documents and len(documents) > 0:
-    try:
-      store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
-      db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
-          os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
-          "zip",
-          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
-      )
-      return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
-    except Exception as e:
-      await _cleanup_directory(store_path)
-      return VectorDbResponse(success = False, error = str(e))
-    finally:
-      del documents
-      gc.collect()
-  else:
-    _error = "No documents found in the knowledgebase folder"
-    logging.warning(_error)
-    return VectorDbResponse(success = False, error = _error)
-
-async def kb_stream_file(filename: str):
-    file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
-    if not os.path.isfile(file_path):
-        raise HTTPException(status_code=404, detail="File not found")
-    def iter_file():
-        with open(file_path, mode="rb") as file:
-            while chunk := file.read(1024*8):
-                yield chunk
-    return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})
+import os, gc, shutil, logging, traceback
+import asyncio, aiofiles, aiofiles.os
+from fastapi import HTTPException
+from fastapi.responses import StreamingResponse
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from ws_bom_robot_app.llm.models.api import RulesRequest, KbRequest, VectorDbResponse
+from ws_bom_robot_app.llm.vector_store.integration.manager import IntegrationManager
+from ws_bom_robot_app.llm.vector_store.db.manager import VectorDbManager
+from ws_bom_robot_app.config import config
+from ws_bom_robot_app.llm.models.kb import load_endpoints
+from ws_bom_robot_app.llm.utils.download import download_files
+
+async def _cleanup_directory(directory_path: str):
+  if os.path.exists(directory_path):
+    await asyncio.to_thread(shutil.rmtree, directory_path)
+
+#@timer
+async def rules(rq: RulesRequest) -> VectorDbResponse:
+  _config = rq.config()
+  db_name = rq.out_name()
+  store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
+  try:
+    await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(),[Document(page_content=rule, metadata={"source": "rules"}) for rule in rq.rules], store_path) #type: ignore
+    db_file_path = shutil.make_archive(os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name), "zip", store_path)
+    return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
+  except Exception as e:
+    try:
+      await _cleanup_directory(store_path)
+    finally:
+      return VectorDbResponse(success = False, error = str(e))
+  finally:
+    gc.collect()
+
+#@atimer
+async def kb(rq: KbRequest) -> VectorDbResponse:
+  os.environ['MPLCONFIGDIR'] = './tmp/.matplotlib'
+  _config = rq.config()
+  db_name = rq.out_name()
+  src_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_src)
+  working_path = os.path.join(src_path, db_name)
+
+  if all([not rq.files,not rq.endpoints,not rq.integrations]):
+    return VectorDbResponse(success = False, error = "No files, endpoints or integrations provided")
+  else:
+    await aiofiles.os.makedirs(src_path, exist_ok=True)
+    await aiofiles.os.makedirs(working_path, exist_ok=True)
+
+  documents: list[Document] = []
+  # Download/copy all files
+  if rq.files:
+    try:
+      loaders = Loader(working_path)
+      filter_file_extensions = loaders.managed_file_extensions()
+      files_to_download = [file for file in rq.files if not os.path.exists(os.path.join(src_path, os.path.basename(file)))]
+      if files_to_download:
+        await download_files(
+          [f"{_config.robot_cms_host}/{_config.robot_cms_kb_folder}/{os.path.basename(file)}" for file in files_to_download if any([file.endswith(ext) for ext in filter_file_extensions])],
+          src_path, authorization=_config.robot_cms_auth)
+      # copy files to working tmp folder
+      for file in rq.files:
+        async with aiofiles.open(os.path.join(src_path, os.path.basename(file)), 'rb') as src_file:
+          async with aiofiles.open(os.path.join(working_path, os.path.basename(file)), 'wb') as dest_file:
+            await dest_file.write(await src_file.read())
+      #load files
+      try:
+        documents.extend(await loaders.load())
+      except Exception as e:
+        tb = traceback.format_exc()
+        _error = f"File loader failure: {e} | {tb}"
+        logging.warning(_error)
+        return VectorDbResponse(success = False, error = _error)
+    except Exception as e:
+      await _cleanup_directory(working_path)
+      return VectorDbResponse(success = False, error = f"Failed to download file {e}")
+
+  if rq.endpoints:
+    try:
+      documents.extend(await load_endpoints(rq.endpoints, working_path))
+    except Exception as e:
+      await _cleanup_directory(working_path)
+      tb = traceback.format_exc()
+      _error = f"Endpoint failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success = False, error = _error)
+
+  if rq.integrations:
+    tasks = []
+    for integration in rq.integrations:
+      tasks.append(
+        IntegrationManager
+        .get_strategy(integration.type.lower(), working_path, integration.__pydantic_extra__) #type: ignore
+        .load()
+      )
+    try:
+      integration_documents = await asyncio.gather(*tasks)
+      for docs in integration_documents:
+        documents.extend(docs)
+    except Exception as e:
+      await _cleanup_directory(working_path)
+      tb = traceback.format_exc()
+      _error = f"Integration failure: {e} | {tb}"
+      logging.warning(_error)
+      return VectorDbResponse(success=False, error=_error)
+
+  #cleanup
+  await _cleanup_directory(working_path)
+
+  if documents and len(documents) > 0:
+    try:
+      store_path = os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_store, db_name)
+      db_file_path = await aiofiles.os.wrap(shutil.make_archive)(
+          os.path.join(_config.robot_data_folder, _config.robot_data_db_folder, _config.robot_data_db_folder_out, db_name),
+          "zip",
+          await VectorDbManager.get_strategy(rq.vector_type).create(rq.embeddings(), documents, store_path, return_folder_path=True)
+      )
+      return VectorDbResponse(file = os.path.basename(db_file_path), vector_type=rq.vector_type)
+    except Exception as e:
+      await _cleanup_directory(store_path)
+      return VectorDbResponse(success = False, error = str(e))
+    finally:
+      del documents
+      gc.collect()
+  else:
+    _error = "No documents found in the knowledgebase folder"
+    logging.warning(_error)
+    return VectorDbResponse(success = False, error = _error)
+
+async def kb_stream_file(filename: str):
+    file_path = os.path.join(config.robot_data_folder, config.robot_data_db_folder, config.robot_data_db_folder_out, filename)
+    if not os.path.isfile(file_path):
+        raise HTTPException(status_code=404, detail="File not found")
+    def iter_file():
+        with open(file_path, mode="rb") as file:
+            while chunk := file.read(1024*8):
+                yield chunk
+    return StreamingResponse(iter_file(), media_type="application/octet-stream", headers={"Content-Disposition": f"attachment; filename={filename}"})

ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,9 +1,10 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceIndexer, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.pipeline.pipeline import Pipeline
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import Optional, Union
+from typing import List, Optional, Union
 from pydantic import BaseModel, Field, AliasChoices
 
 class ConfluenceParams(BaseModel):
@@ -16,6 +17,7 @@ class ConfluenceParams(BaseModel):
     password: Confluence password or Cloud API token, if filled, set the access_token to None and vice versa.
     access_token (str): The personal access token for authenticating with Confluence, e.g., 'AT....'
     spaces (list[str]): A list of Confluence spaces to interact with, e.g., ['SPACE1', 'SPACE2'].
+    max_num_of_docs_from_each_space (int): The maximum number of documents to fetch from each space. Defaults to 500, with a maximum limit of 5000.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g., ['.pdf', '.docx'].
   """
   url: str
@@ -23,6 +25,7 @@ class ConfluenceParams(BaseModel):
   password: Optional[str] = None
   access_token: Optional[str] = Field(None, validation_alias=AliasChoices("accessToken","access_token"))
   spaces: list[str] = []
+  max_num_of_docs_from_each_space: int = Field(default=500, ge=1, le=5000,validation_alias=AliasChoices("maxNumOfDocsFromEachSpace","max_num_of_docs_from_each_space"))
   extension: list[str] = Field(default=None)
 class Confluence(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
@@ -33,7 +36,8 @@ class Confluence(IntegrationStrategy):
     return 'confluence'
   def run(self) -> None:
     indexer_config = ConfluenceIndexerConfig(
-      spaces=self.__data.spaces
+      spaces=self.__data.spaces,
+      max_num_of_docs_from_each_space=self.__data.max_num_of_docs_from_each_space
     )
     downloader_config = ConfluenceDownloaderConfig(
       download_dir=self.working_directory
@@ -43,13 +47,37 @@ class Confluence(IntegrationStrategy):
       url=self.__data.url,
       username=self.__data.username
     )
-    self.__unstructured_ingest.pipeline(
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
       connection_config,
-      extension=self.__data.extension).run()
+      extension=self.__data.extension
+    )
+    pipeline.indexer_step.process = CustomConfluenceIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
+class CustomConfluenceIndexer(ConfluenceIndexer):
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def _get_docs_ids_within_one_space(self, space_key: str) -> List[dict]:
+      with self.connection_config.get_client() as client:
+          pages = client.get_all_pages_from_space(
+              space=space_key,
+              start=0,
+              limit=self.index_config.max_num_of_docs_from_each_space, #explicitly limit the number of pages fetched (omitted  in unstructured-ingest)
+              expand=None,
+              content_type="page",  # blogpost and comment types not currently supported
+              status=None,
+          )
+      limited_pages = pages[: self.index_config.max_num_of_docs_from_each_space]
+      doc_ids = [{"space_id": space_key, "doc_id": page["id"]} for page in limited_pages]
+      return doc_ids
+

ws_bom_robot_app/llm/vector_store/integration/shopify.py

@@ -1,143 +1,143 @@
-import asyncio, logging, aiohttp
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from langchain_core.documents import Document
-from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import List, Union, Optional
-from pydantic import BaseModel, Field, AliasChoices, field_validator
-import json
-import os
-
-class ShopifyParams(BaseModel):
-  """
-  ShopifyParams is a model that defines the parameters required for Shopify integration.
-
-  Attributes:
-    shop_name (str): The shop name for Shopify.
-    access_token (str): The access token for Shopify.
-    graphql_query (Union[str, dict]): The GraphQL query string or dict for Shopify.
-  """
-  shop_name: str = Field(validation_alias=AliasChoices("shopName","shop_name"))
-  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
-  graphql_query: Union[str, dict] = Field(validation_alias=AliasChoices("graphqlQuery","graphql_query"))
-
-  @field_validator('graphql_query')
-  @classmethod
-  def extract_query_string(cls, v):
-    """Extract the query string from dict format if needed"""
-    if isinstance(v, dict) and 'query' in v:
-      return v['query']
-    return v
-
-class Shopify(IntegrationStrategy):
-  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
-    super().__init__(knowledgebase_path, data)
-    self.__data = ShopifyParams.model_validate(self.data)
-
-  def working_subdirectory(self) -> str:
-    return 'shopify'
-
-  async def run(self) -> None:
-    _data = await self.__get_data()
-    json_file_path = os.path.join(self.working_directory, 'shopify_data.json')
-    with open(json_file_path, 'w', encoding='utf-8') as f:
-      json.dump(_data, f, ensure_ascii=False)
-
-  async def load(self) -> list[Document]:
-    await self.run()
-    await asyncio.sleep(1)
-    return await Loader(self.working_directory).load()
-
-  async def __get_data(self, page_size: int = 50) -> List[dict]:
-      # URL dell'API
-    url = f"https://{self.__data.shop_name}.myshopify.com/admin/api/2024-07/graphql.json"
-
-    # Headers
-    headers = {
-        "X-Shopify-Access-Token": self.__data.access_token,
-        "Content-Type": "application/json"
-    }
-
-    all_products: List[dict] = []
-    has_next_page = True
-    cursor = None
-    retry_count = 0
-    max_retries = 5
-
-    while has_next_page:
-        # Variables per la query
-        variables = {
-            "first": page_size
-        }
-
-        if cursor:
-            variables["after"] = cursor
-
-        # Payload della richiesta
-        payload = {
-            "query": self.__data.graphql_query,
-            "variables": variables
-        }
-
-        try:
-            # Effettua la richiesta
-            async with aiohttp.ClientSession() as session:
-              async with session.post(url, headers=headers, json=payload) as response:
-                # Controlla se la risposta è JSON
-                try:
-                  data = await response.json()
-                except aiohttp.ContentTypeError:
-                  text = await response.text()
-                  logging.error(f"Non-JSON response received. Status code: {response.status}")
-                  logging.error(f"Content: {text}")
-                  raise Exception("Invalid response from API")
-
-            # Gestione del throttling
-            if "errors" in data:
-                error = data["errors"][0]
-                if error.get("extensions", {}).get("code") == "THROTTLED":
-                    retry_count += 1
-                    if retry_count > max_retries:
-                        raise Exception("Too many throttling attempts. Stopping execution.")
-
-                    # Aspetta un po' più a lungo ad ogni tentativo
-                    wait_time = 2 ** retry_count  # Backoff esponenziale
-                    print(f"Rate limit reached. Waiting {wait_time} seconds... (Attempt {retry_count}/{max_retries})")
-                    await asyncio.sleep(wait_time)
-                    continue
-                else:
-                    raise Exception(f"GraphQL errors: {data['errors']}")
-
-            # Resetta il contatore dei retry se la richiesta è andata bene
-            retry_count = 0
-
-            # Estrae i dati
-            products_data = data["data"]["products"]
-            edges = products_data["edges"]
-            page_info = products_data["pageInfo"]
-
-            # Aggiungi i prodotti alla lista
-            for edge in edges:
-                all_products.append(edge["node"])
-
-            # Aggiorna il cursore e il flag per la paginazione
-            has_next_page = page_info["hasNextPage"]
-            cursor = page_info["endCursor"]
-
-            print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_products)}")
-
-            # Piccola pausa per evitare di saturare l'API
-            await asyncio.sleep(0.1)
-
-        except aiohttp.ClientError as e:
-            logging.error(f"Connection error: {e}")
-            retry_count += 1
-            if retry_count <= max_retries:
-                wait_time = 2 ** retry_count
-                logging.warning(f"Retrying in {wait_time} seconds...")
-                await asyncio.sleep(wait_time)
-                continue
-            else:
-                raise Exception("Too many network errors. Stopping execution.")
-
-    logging.info(f"Data retrieval completed! Total products: {len(all_products)}")
-    return all_products
+import asyncio, logging, aiohttp
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import List, Union, Optional
+from pydantic import BaseModel, Field, AliasChoices, field_validator
+import json
+import os
+
+class ShopifyParams(BaseModel):
+  """
+  ShopifyParams is a model that defines the parameters required for Shopify integration.
+
+  Attributes:
+    shop_name (str): The shop name for Shopify.
+    access_token (str): The access token for Shopify.
+    graphql_query (Union[str, dict]): The GraphQL query string or dict for Shopify.
+  """
+  shop_name: str = Field(validation_alias=AliasChoices("shopName","shop_name"))
+  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
+  graphql_query: Union[str, dict] = Field(validation_alias=AliasChoices("graphqlQuery","graphql_query"))
+
+  @field_validator('graphql_query')
+  @classmethod
+  def extract_query_string(cls, v):
+    """Extract the query string from dict format if needed"""
+    if isinstance(v, dict) and 'query' in v:
+      return v['query']
+    return v
+
+class Shopify(IntegrationStrategy):
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = ShopifyParams.model_validate(self.data)
+
+  def working_subdirectory(self) -> str:
+    return 'shopify'
+
+  async def run(self) -> None:
+    _data = await self.__get_data()
+    json_file_path = os.path.join(self.working_directory, 'shopify_data.json')
+    with open(json_file_path, 'w', encoding='utf-8') as f:
+      json.dump(_data, f, ensure_ascii=False)
+
+  async def load(self) -> list[Document]:
+    await self.run()
+    await asyncio.sleep(1)
+    return await Loader(self.working_directory).load()
+
+  async def __get_data(self, page_size: int = 50) -> List[dict]:
+      # URL dell'API
+    url = f"https://{self.__data.shop_name}.myshopify.com/admin/api/2024-07/graphql.json"
+
+    # Headers
+    headers = {
+        "X-Shopify-Access-Token": self.__data.access_token,
+        "Content-Type": "application/json"
+    }
+
+    all_products: List[dict] = []
+    has_next_page = True
+    cursor = None
+    retry_count = 0
+    max_retries = 5
+
+    while has_next_page:
+        # Variables per la query
+        variables = {
+            "first": page_size
+        }
+
+        if cursor:
+            variables["after"] = cursor
+
+        # Payload della richiesta
+        payload = {
+            "query": self.__data.graphql_query,
+            "variables": variables
+        }
+
+        try:
+            # Effettua la richiesta
+            async with aiohttp.ClientSession() as session:
+              async with session.post(url, headers=headers, json=payload) as response:
+                # Controlla se la risposta è JSON
+                try:
+                  data = await response.json()
+                except aiohttp.ContentTypeError:
+                  text = await response.text()
+                  logging.error(f"Non-JSON response received. Status code: {response.status}")
+                  logging.error(f"Content: {text}")
+                  raise Exception("Invalid response from API")
+
+            # Gestione del throttling
+            if "errors" in data:
+                error = data["errors"][0]
+                if error.get("extensions", {}).get("code") == "THROTTLED":
+                    retry_count += 1
+                    if retry_count > max_retries:
+                        raise Exception("Too many throttling attempts. Stopping execution.")
+
+                    # Aspetta un po' più a lungo ad ogni tentativo
+                    wait_time = 2 ** retry_count  # Backoff esponenziale
+                    print(f"Rate limit reached. Waiting {wait_time} seconds... (Attempt {retry_count}/{max_retries})")
+                    await asyncio.sleep(wait_time)
+                    continue
+                else:
+                    raise Exception(f"GraphQL errors: {data['errors']}")
+
+            # Resetta il contatore dei retry se la richiesta è andata bene
+            retry_count = 0
+
+            # Estrae i dati
+            products_data = data["data"]["products"]
+            edges = products_data["edges"]
+            page_info = products_data["pageInfo"]
+
+            # Aggiungi i prodotti alla lista
+            for edge in edges:
+                all_products.append(edge["node"])
+
+            # Aggiorna il cursore e il flag per la paginazione
+            has_next_page = page_info["hasNextPage"]
+            cursor = page_info["endCursor"]
+
+            print(f"Recuperati {len(edges)} prodotti. Totale: {len(all_products)}")
+
+            # Piccola pausa per evitare di saturare l'API
+            await asyncio.sleep(0.1)
+
+        except aiohttp.ClientError as e:
+            logging.error(f"Connection error: {e}")
+            retry_count += 1
+            if retry_count <= max_retries:
+                wait_time = 2 ** retry_count
+                logging.warning(f"Retrying in {wait_time} seconds...")
+                await asyncio.sleep(wait_time)
+                continue
+            else:
+                raise Exception("Too many network errors. Stopping execution.")
+
+    logging.info(f"Data retrieval completed! Total products: {len(all_products)}")
+    return all_products

ws_bom_robot_app/llm/vector_store/integration/sitemap.py

@@ -33,6 +33,7 @@ class Sitemap(IntegrationStrategy):
         self.__exclude_class: list[str] = self.data.get("excludeClass",[]) # type: ignore
         self.__exclude_id: list[str] = self.data.get("excludeId",[]) # type: ignore
         self.__restrict_to_same_domain: bool = self.data.get("restrictDomain", True) # type: ignore
+        self.__header_template = self.data.get("headers", None)
     def working_subdirectory(self) -> str:
         return ""
     def _extract(self, tag: Tag) -> str:
@@ -81,7 +82,8 @@ class Sitemap(IntegrationStrategy):
                 filter_urls=self.__filter_urls,
                 parsing_function=self._parse,
                 is_local=self._is_local(self.__sitemap_url),
-                restrict_to_same_domain=self.__restrict_to_same_domain
+                restrict_to_same_domain=self.__restrict_to_same_domain,
+                header_template=self.__header_template
             )
             _docs = self._output([document async for document in self.alazy_load(_loader)])
             if self._is_local(self.__sitemap_url):