ws-bom-robot-app 0.0.87__py3-none-any.whl → 0.0.88__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,9 +1,10 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceIndexer, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.pipeline.pipeline import Pipeline
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import Optional, Union
+from typing import List, Optional, Union
 from pydantic import BaseModel, Field, AliasChoices
 
 class ConfluenceParams(BaseModel):
@@ -16,6 +17,7 @@ class ConfluenceParams(BaseModel):
     password: Confluence password or Cloud API token, if filled, set the access_token to None and vice versa.
     access_token (str): The personal access token for authenticating with Confluence, e.g., 'AT....'
     spaces (list[str]): A list of Confluence spaces to interact with, e.g., ['SPACE1', 'SPACE2'].
+    max_num_of_docs_from_each_space (int): The maximum number of documents to fetch from each space. Defaults to 500, with a maximum limit of 5000.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g., ['.pdf', '.docx'].
   """
   url: str
@@ -23,6 +25,7 @@ class ConfluenceParams(BaseModel):
   password: Optional[str] = None
   access_token: Optional[str] = Field(None, validation_alias=AliasChoices("accessToken","access_token"))
   spaces: list[str] = []
+  max_num_of_docs_from_each_space: int = Field(default=500, ge=1, le=5000,validation_alias=AliasChoices("maxNumOfDocsFromEachSpace","max_num_of_docs_from_each_space"))
   extension: list[str] = Field(default=None)
 class Confluence(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
@@ -33,7 +36,8 @@ class Confluence(IntegrationStrategy):
     return 'confluence'
   def run(self) -> None:
     indexer_config = ConfluenceIndexerConfig(
-      spaces=self.__data.spaces
+      spaces=self.__data.spaces,
+      max_num_of_docs_from_each_space=self.__data.max_num_of_docs_from_each_space
     )
     downloader_config = ConfluenceDownloaderConfig(
       download_dir=self.working_directory
@@ -43,13 +47,37 @@ class Confluence(IntegrationStrategy):
       url=self.__data.url,
       username=self.__data.username
     )
-    self.__unstructured_ingest.pipeline(
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
       connection_config,
-      extension=self.__data.extension).run()
+      extension=self.__data.extension
+    )
+    pipeline.indexer_step.process = CustomConfluenceIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
+class CustomConfluenceIndexer(ConfluenceIndexer):
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def _get_docs_ids_within_one_space(self, space_key: str) -> List[dict]:
+      with self.connection_config.get_client() as client:
+          pages = client.get_all_pages_from_space(
+              space=space_key,
+              start=0,
+              limit=self.index_config.max_num_of_docs_from_each_space, #explicitly limit the number of pages fetched (omitted  in unstructured-ingest)
+              expand=None,
+              content_type="page",  # blogpost and comment types not currently supported
+              status=None,
+          )
+      limited_pages = pages[: self.index_config.max_num_of_docs_from_each_space]
+      doc_ids = [{"space_id": space_key, "doc_id": page["id"]} for page in limited_pages]
+      return doc_ids
+

{ws_bom_robot_app-0.0.87.dist-info → ws_bom_robot_app-0.0.88.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ws_bom_robot_app
-Version: 0.0.87
+Version: 0.0.88
 Summary: A FastAPI application serving ws bom/robot/llm platform ai.
 Home-page: https://github.com/websolutespa/bom
 Author: Websolute Spa

{ws_bom_robot_app-0.0.87.dist-info → ws_bom_robot_app-0.0.88.dist-info}/RECORD

@@ -51,7 +51,7 @@ ws_bom_robot_app/llm/vector_store/db/qdrant.py,sha256=HfEtFqMF0wIn5SNbst6glw7gG4
 ws_bom_robot_app/llm/vector_store/integration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ws_bom_robot_app/llm/vector_store/integration/azure.py,sha256=OEa96Dlf1CX0tjrTjX4KP3D_HTn249ukc9sluPbdOyU,3389
 ws_bom_robot_app/llm/vector_store/integration/base.py,sha256=4zI1TtacyVw0jcY0wFv_4y93iX2cgFGn6rAGXd-nSxk,4331
-ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=UwnDR-TG1jq1Z4bQomY7XnUpPNjWKtuR8ZfTQ7xsajs,2722
+ws_bom_robot_app/llm/vector_store/integration/confluence.py,sha256=TMmGe53tHRTgHJ7nA8DqZVodo3aMEzHrrSdl0-I0-S0,4350
 ws_bom_robot_app/llm/vector_store/integration/dropbox.py,sha256=vDEVTq7xkXNvpirMkJHm90WzxcSQqCXNc8PBwzLvSH4,2626
 ws_bom_robot_app/llm/vector_store/integration/gcs.py,sha256=P-NKwNag6fkY3bzFvVkAK5Ayl5CKM8T0MvkaFFwSyT0,3181
 ws_bom_robot_app/llm/vector_store/integration/github.py,sha256=1J4Ph3s58ngEIH5HyCMeeD6lVo2GzdU8y41BvPSLZcc,2441
@@ -69,7 +69,7 @@ ws_bom_robot_app/llm/vector_store/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5
 ws_bom_robot_app/llm/vector_store/loader/base.py,sha256=GjUS2oaz0LHOSal5pipBkomZtrYUNcKPSd8bzhUU5Dc,6889
 ws_bom_robot_app/llm/vector_store/loader/docling.py,sha256=IOv1A0HSIWiHWQFzI4fdApfxrKgXOqwmC3mPXlKplqQ,4012
 ws_bom_robot_app/llm/vector_store/loader/json_loader.py,sha256=LDppW0ZATo4_1hh-KlsAM3TLawBvwBxva_a7k5Oz1sc,858
-ws_bom_robot_app-0.0.87.dist-info/METADATA,sha256=3pNNKhvD9yn8STxW00dHyEGUgedYB_bSKzK7ioNkmZ0,9985
-ws_bom_robot_app-0.0.87.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ws_bom_robot_app-0.0.87.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
-ws_bom_robot_app-0.0.87.dist-info/RECORD,,
+ws_bom_robot_app-0.0.88.dist-info/METADATA,sha256=31SMpndBYfgv0PylO85JpVzD13y9bqTDgXD-1_aHAug,9985
+ws_bom_robot_app-0.0.88.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ws_bom_robot_app-0.0.88.dist-info/top_level.txt,sha256=Yl0akyHVbynsBX_N7wx3H3ZTkcMLjYyLJs5zBMDAKcM,17
+ws_bom_robot_app-0.0.88.dist-info/RECORD,,