ws-bom-robot-app 0.0.37__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,9 +1,10 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceIndexer, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.pipeline.pipeline import Pipeline
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import Union
+from typing import List, Optional, Union
 from pydantic import BaseModel, Field, AliasChoices
 
 class ConfluenceParams(BaseModel):
@@ -12,15 +13,19 @@ class ConfluenceParams(BaseModel):
 
   Attributes:
     url (str): The URL of the Confluence instance, e.g., 'https://example.atlassian.net'.
-    access_token (str): The access token for authenticating with Confluence, e.g., 'AT....'
-    user_email (str): The email address of the Confluence user
+    username (str): The email address or username of the Confluence user
+    password: Confluence password or Cloud API token, if filled, set the access_token to None and vice versa.
+    access_token (str): The personal access token for authenticating with Confluence, e.g., 'AT....'
     spaces (list[str]): A list of Confluence spaces to interact with, e.g., ['SPACE1', 'SPACE2'].
+    max_num_of_docs_from_each_space (int): The maximum number of documents to fetch from each space. Defaults to 500, with a maximum limit of 5000.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g., ['.pdf', '.docx'].
   """
   url: str
-  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
-  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
+  username: str = Field(validation_alias=AliasChoices("userName","userEmail","username"))
+  password: Optional[str] = None
+  access_token: Optional[str] = Field(None, validation_alias=AliasChoices("accessToken","access_token"))
   spaces: list[str] = []
+  max_num_of_docs_from_each_space: int = Field(default=500, ge=1, le=5000,validation_alias=AliasChoices("maxNumOfDocsFromEachSpace","max_num_of_docs_from_each_space"))
   extension: list[str] = Field(default=None)
 class Confluence(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
@@ -31,23 +36,48 @@ class Confluence(IntegrationStrategy):
     return 'confluence'
   def run(self) -> None:
     indexer_config = ConfluenceIndexerConfig(
-      spaces=self.__data.spaces
+      spaces=self.__data.spaces,
+      max_num_of_docs_from_each_space=self.__data.max_num_of_docs_from_each_space
     )
     downloader_config = ConfluenceDownloaderConfig(
       download_dir=self.working_directory
     )
     connection_config = ConfluenceConnectionConfig(
-      access_config=ConfluenceAccessConfig(api_token=self.__data.access_token),
+      access_config=ConfluenceAccessConfig(password=self.__data.password, token=self.__data.access_token),
       url=self.__data.url,
-      user_email=self.__data.user_email
+      username=self.__data.username
     )
-    self.__unstructured_ingest.pipeline(
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
       connection_config,
-      extension=self.__data.extension).run()
+      extension=self.__data.extension
+    )
+    pipeline.indexer_step.process = CustomConfluenceIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
+class CustomConfluenceIndexer(ConfluenceIndexer):
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def _get_docs_ids_within_one_space(self, space_key: str) -> List[dict]:
+      with self.connection_config.get_client() as client:
+          pages = client.get_all_pages_from_space(
+              space=space_key,
+              start=0,
+              limit=self.index_config.max_num_of_docs_from_each_space, #explicitly limit the number of pages fetched (omitted  in unstructured-ingest)
+              expand=None,
+              content_type="page",  # blogpost and comment types not currently supported
+              status=None,
+          )
+      limited_pages = pages[: self.index_config.max_num_of_docs_from_each_space]
+      doc_ids = [{"space_id": space_key, "doc_id": page["id"]} for page in limited_pages]
+      return doc_ids
+

ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union

ws_bom_robot_app/llm/vector_store/integration/gcs.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
+from unstructured_ingest.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -1,10 +1,12 @@
 import asyncio
 from typing import Optional, Union
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.git import GitAccessConfig
-from unstructured_ingest.connector.github import SimpleGitHubConfig
-from unstructured_ingest.runner import GithubRunner
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.processes.connectors.github import (
+    GithubIndexerConfig,
+    GithubDownloaderConfig,
+    GithubConnectionConfig,
+    GithubAccessConfig
+)
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
@@ -27,28 +29,26 @@ class Github(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = GithubParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'github'
   def run(self) -> None:
-    access_config = GitAccessConfig(
-      access_token=self.__data.access_token
-    )
-    file_ext = self.__data.file_ext or None
-    file_glob = [f"**/*{ext}" for ext in file_ext] if file_ext else None
-    config = SimpleGitHubConfig(
-      url = self.__data.repo,
-      access_config=access_config,
+    indexer_config = GithubIndexerConfig(
       branch=self.__data.branch,
-      file_glob=file_glob
+      recursive=True
+    )
+    downloader_config = GithubDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = GithubConnectionConfig(
+      access_config=GithubAccessConfig(access_token=self.__data.access_token),
+      url=self.__data.repo
     )
-    runner = GithubRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
-      )
-    runner.run()
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.file_ext).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,10 +1,38 @@
 import asyncio
+import json
+from pathlib import Path
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.data_types.file_data import FileData as OriginalFileData, BatchFileData as OriginalBatchFileData
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
+
+# UTF-8 safe FileData classes
+class FileData(OriginalFileData):
+  @classmethod
+  def from_file(cls, path: str):
+    path = Path(path).resolve()
+    if not path.exists() or not path.is_file():
+      raise ValueError(f"file path not valid: {path}")
+    for encoding in ['utf-8', 'cp1252', 'iso-8859-1', 'latin-1']:
+      try:
+        with open(str(path), "r", encoding=encoding) as f:
+          return cls.model_validate(json.load(f))
+      except (UnicodeDecodeError, UnicodeError):
+        continue
+    raise ValueError(f"Could not decode file {path} with any supported encoding")
+
+  def to_file(self, path: str) -> None:
+    path = Path(path).resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(str(path), "w", encoding="utf-8") as f:
+      json.dump(self.model_dump(), f, indent=2, ensure_ascii=False)
+
+class BatchFileData(OriginalBatchFileData, FileData):
+  pass
+
 class GoogleDriveParams(BaseModel):
   """
   GoogleDriveParams is a model that holds parameters for Google Drive integration.
@@ -42,26 +70,27 @@ class GoogleDrive(IntegrationStrategy):
     super().__init__(knowledgebase_path, data)
     self.__data = GoogleDriveParams.model_validate(self.data)
     self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+    self._apply_encoding_fix()
+
+  def _apply_encoding_fix(self):
+    """Replace FileData classes with UTF-8 safe versions"""
+    import unstructured_ingest.data_types.file_data as fd
+    fd.FileData = FileData
+    fd.BatchFileData = BatchFileData
+    fd.file_data_from_file = lambda path: BatchFileData.from_file(path) if path else FileData.from_file(path)
+
   def working_subdirectory(self) -> str:
     return 'googledrive'
+
   def run(self) -> None:
-    indexer_config = GoogleDriveIndexerConfig(
-      extensions=self.__data.extensions,
-      recursive=self.__data.recursive
-    )
-    downloader_config = GoogleDriveDownloaderConfig(
-      download_dir=self.working_directory
-    )
-    connection_config = GoogleDriveConnectionConfig(
-      access_config=GoogleDriveAccessConfig(
-        service_account_key=self.__data.service_account_key
-        ),
-      drive_id=self.__data.drive_id
-    )
     self.__unstructured_ingest.pipeline(
-      indexer_config,
-      downloader_config,
-      connection_config).run()
+      GoogleDriveIndexerConfig(extensions=self.__data.extensions, recursive=self.__data.recursive),
+      GoogleDriveDownloaderConfig(download_dir=self.working_directory),
+      GoogleDriveConnectionConfig(
+        access_config=GoogleDriveAccessConfig(service_account_key=self.__data.service_account_key),
+        drive_id=self.__data.drive_id
+      )
+    ).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

ws_bom_robot_app/llm/vector_store/integration/jira.py

@@ -1,114 +1,151 @@
-import asyncio
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.jira import SimpleJiraConfig, JiraAccessConfig
-from unstructured_ingest.runner import JiraRunner
+import logging
+import asyncio, os
+import sys
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
-from typing import Optional, Union
-import requests
-import unstructured_ingest.connector.jira
+from typing import Any, Generator, Iterable, Optional, Union
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.connectors.jira import (
+    JiraIndexerConfig,
+    JiraIndexer,
+    JiraIssueMetadata,
+    api_page_based_generator,
+    JiraDownloaderConfig,
+    JiraDownloader,
+    DEFAULT_C_SEP,
+    DEFAULT_R_SEP,
+    JiraConnectionConfig,
+    JiraAccessConfig
+)
+from unstructured_ingest.pipeline.pipeline import (
+  Pipeline,
+  PartitionerConfig,
+  FiltererConfig
+)
+from unstructured_ingest.interfaces import ProcessorConfig
 
 class JiraParams(BaseModel):
   """
   JiraParams is a Pydantic model that represents the parameters required to interact with a Jira instance.
+  Docs: https://docs.unstructured.io/open-source/ingestion/source-connectors/jira#jira
 
   Attributes:
     url (str): The URL of the Jira instance, e.g., 'https://example.atlassian.net'.
-    access_token (str): The access token for authenticating with the Jira API.
+    access_token (str): The access token for authenticating with the Jira API: https://id.atlassian.com/manage-profile/security/api-tokens
     user_email (str): The email address of the Jira user.
     projects (list[str]): A list of project keys or IDs to interact with, e.g., ['SCRUM', 'PROJ1'].
     boards (Optional[list[str]]): An optional list of board IDs to interact with. Defaults to None, e.g., ['1', '2'].
     issues (Optional[list[str]]): An optional list of issue keys or IDs to interact with. Defaults to None, e.g., ['SCRUM-1', 'PROJ1-1'].
   """
-  url: str
-  access_token: str = Field(validation_alias=AliasChoices("accessToken","access_token"))
-  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"))
+  url: str = Field(..., pattern=r'^https?:\/\/.+')
+  access_token: str = Field(..., validation_alias=AliasChoices("accessToken","access_token"), min_length=1)
+  user_email: str = Field(validation_alias=AliasChoices("userEmail","user_email"), min_length=1)
   projects: list[str]
   boards: Optional[list[str]] | None = None
   issues: Optional[list[str]] | None = None
-  fieldsMappingUrl: Optional[str] | None = None
+  status_filters: Optional[list[str]] | None = None
 
 class Jira(IntegrationStrategy):
-  DEFAULT_C_SEP = " " * 5
-  DEFAULT_R_SEP = "\n"
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = JiraParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'jira'
   def run(self) -> None:
-    unstructured_ingest.connector.jira._get_dropdown_fields_for_issue = self._get_dropdown_fields_for_issue
-    access_config = JiraAccessConfig(
-      api_token=self.__data.access_token
-    )
-    config = SimpleJiraConfig(
-      user_email=self.__data.user_email,
-      url = self.__data.url,
-      access_config=access_config,
+    indexer_config = JiraIndexerConfig(
       projects=self.__data.projects,
       boards=self.__data.boards,
-      issues=self.__data.issues
-    )
-    runner = JiraRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
+      issues=self.__data.issues,
+      status_filters=self.__data.status_filters
       )
-    runner.run()
+    downloader_config = JiraDownloaderConfig(
+      download_dir=self.working_directory,
+      download_attachments=False
+    )
+    _is_cloud = "atlassian.net" in self.__data.url
+    _access_config = JiraAccessConfig(token=self.__data.access_token) \
+      if not _is_cloud  \
+      else JiraAccessConfig(password=self.__data.access_token)
+    connection_config = JiraConnectionConfig(
+      access_config=_access_config,
+      username=self.__data.user_email,
+      url=self.__data.url,
+      cloud=_is_cloud
+    )
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=None)
+    if _is_cloud and sys.platform == "win32":
+      pipeline.indexer_step.process = CustomJiraIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.downloader_step.process = CustomJiraDownloader(**vars(pipeline.downloader_step.process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
-  def _remap_custom_fields(self, field_list):
-    auth = (self.__data.user_email, self.__data.access_token)
-    response = requests.get(self.__data.fieldsMappingUrl, auth=auth)
 
-    if response.status_code == 200:
-      mapper: dict = response.json()
-    remapped_field_list = {}
-    for field_key, field_value in field_list.items():
-        new_key = None
-        for map_item in mapper:
-            if field_key == map_item["id"]:
-                # Usa il nome mappato come nuova chiave
-                new_key = map_item["name"]
-                break
-
-        if new_key is None:
-            new_key = field_key
+# region override
+class CustomJiraIndexer(JiraIndexer):
+  """
+    fix default run_jql for cloud: missing enhanced_jql
+  """
+  import sys
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def run_jql(self, jql: str, **kwargs) -> Generator[JiraIssueMetadata, None, None]:
+      with self.connection_config.get_client() as client:
+          for issue in api_page_based_generator(client.jql, jql=jql, **kwargs):
+              yield JiraIssueMetadata.model_validate(issue)
 
-        remapped_field_list[new_key] = field_value
+class CustomJiraDownloader(JiraDownloader):
+  CUSTOM_FIELDS: list | None = None
+  def _set_custom_fields(self) -> list:
+    with self.connection_config.get_client() as client:
+        _custom_fields = client.get_all_custom_fields()
+        return [{"id": item["id"], "name": item["name"]} for item in _custom_fields]
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+    if not self.CUSTOM_FIELDS:
+      self.CUSTOM_FIELDS = self._set_custom_fields()
 
-    return remapped_field_list
+  def _get_custom_fields_for_issue(self, issue: dict, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP) -> str:
+      def _parse_value(value: Any) -> Any:
+          if isinstance(value, dict):
+            _candidate = ["displayName", "name", "value"]
+            for item in _candidate:
+                if item in value:
+                    return value[item]
+          return value
+      def _remap_custom_fields(fields: dict):
+        remapped_fields = {}
+        for field_key, field_value in fields.items():
+          new_key = next((map_item["name"] for map_item in self.CUSTOM_FIELDS if field_key == map_item["id"]), field_key)
+          if new_key != field_value:
+            remapped_fields[new_key] = field_value
+        return remapped_fields
+      filtered_fields = {key: _parse_value(value) for key, value in issue.items() if value is not None and type(value) not in [list]}
+      custom_fields =_remap_custom_fields(filtered_fields)
+      return (r_sep + c_sep ).join([f"{key}: {value}{r_sep}" for key, value in custom_fields.items()])
 
-  def _get_dropdown_fields_for_issue(self, issue, c_sep=DEFAULT_C_SEP, r_sep=DEFAULT_R_SEP):
-      all_fields = {}
-      for key, value in issue.items():
-          if value is not None:
-              if isinstance(value, list) and (len(value) > 0):
-                  all_fields[key] = value
-              else:
-                  all_fields[key] = value
-      mapped_fields = self._remap_custom_fields(all_fields)
-      return f"""
-      IssueType:{issue["issuetype"]["name"]}
-      {r_sep}
-      Status:{issue["status"]["name"]}
-      {r_sep}
-      Priority:{issue["priority"]}
-      {r_sep}
-      AssigneeID_Name:{issue["assignee"]["accountId"]}{c_sep}{issue["assignee"]["displayName"]}
-      {r_sep}
-      ReporterAdr_Name:{issue["reporter"]["emailAddress"]}{c_sep}{issue["reporter"]["displayName"]}
-      {r_sep}
-      Labels:{c_sep.join(issue["labels"])}
-      {r_sep}
-      Components:{c_sep.join([component["name"] for component in issue["components"]])}
+  def _get_text_fields_for_issue(self, issue: dict, c_sep: str = DEFAULT_C_SEP, r_sep: str = DEFAULT_R_SEP) -> str:
+      #no need any more: original data will be included in the custom fields
+      #_origin = super()._get_text_fields_for_issue(issue, c_sep=c_sep, r_sep=r_sep)
+      _custom_fields = self._get_custom_fields_for_issue(issue, c_sep=c_sep, r_sep=r_sep)
+      return f"""Details:
       {r_sep}
-      {(r_sep + c_sep ).join([f"{key}:{value}{r_sep}" for key, value in mapped_fields.items()])}
-      """
+      {_custom_fields}"""
+# endregion

ws_bom_robot_app/llm/vector_store/integration/manager.py

@@ -12,7 +12,9 @@ from ws_bom_robot_app.llm.vector_store.integration.sftp import Sftp
 from ws_bom_robot_app.llm.vector_store.integration.sharepoint import Sharepoint
 from ws_bom_robot_app.llm.vector_store.integration.sitemap import Sitemap
 from ws_bom_robot_app.llm.vector_store.integration.slack import Slack
-
+from ws_bom_robot_app.llm.vector_store.integration.thron import Thron
+from ws_bom_robot_app.llm.vector_store.integration.shopify import Shopify
+from ws_bom_robot_app.llm.vector_store.integration.api import Api
 class IntegrationManager:
   _list: dict[str, Type[IntegrationStrategy]] = {
     "llmkbazure": Azure,
@@ -27,7 +29,9 @@ class IntegrationManager:
     "llmkbsharepoint": Sharepoint,
     "llmkbsitemap": Sitemap,
     "llmkbslack": Slack,
-
+    "llmkbthron": Thron,
+    "llmkbshopify": Shopify,
+    "llmkbapi": Api,
   }
   @classmethod
   def get_strategy(cls, name: str, knowledgebase_path: str, data: dict[str, str]) -> IntegrationStrategy:

ws_bom_robot_app/llm/vector_store/integration/s3.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.s3 import S3ConnectionConfig, S3AccessConfig, S3DownloaderConfig, S3IndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.s3 import S3ConnectionConfig, S3AccessConfig, S3DownloaderConfig, S3IndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/sftp.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.sftp import SftpConnectionConfig, SftpAccessConfig, SftpDownloaderConfig, SftpIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/sharepoint.py

@@ -1,7 +1,7 @@
 import asyncio, logging, traceback
 from dataclasses import dataclass
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointIndexer, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
+from unstructured_ingest.processes.connectors.sharepoint  import SharepointIndexerConfig, SharepointIndexer, SharepointDownloaderConfig, SharepointConnectionConfig, SharepointAccessConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional
@@ -14,22 +14,18 @@ class SharepointParams(BaseModel):
   Attributes:
     client_id (str): The client ID for SharePoint authentication.
     client_secret (str): The client secret for SharePoint authentication.
+    tenant_id (str, optional): The tenant ID for SharePoint authentication. Defaults to None.
     site_url (str): The URL of the SharePoint site. i.e. site collection level: https://<tenant>.sharepoint.com/sites/<site-collection-name>, or root site: https://<tenant>.sharepoint.com
     site_path (str, optional): TThe path in the SharePoint site from which to start parsing files, for example "Shared Documents". Defaults to None.
     recursive (bool, optional): Whether to recursively access subdirectories. Defaults to False.
-    omit_files (bool, optional): Whether to omit files from the results. Defaults to False.
-    omit_pages (bool, optional): Whether to omit pages from the results. Defaults to False.
-    omit_lists (bool, optional): Whether to omit lists from the results. Defaults to False.
     extension (list[str], optional): A list of file extensions to include, i.e. [".pdf"]  Defaults to None.
   """
   client_id : str = Field(validation_alias=AliasChoices("clientId","client_id"))
   client_secret : str = Field(validation_alias=AliasChoices("clientSecret","client_secret"))
   site_url: str = Field(validation_alias=AliasChoices("siteUrl","site_url"))
   site_path: str = Field(default=None,validation_alias=AliasChoices("sitePath","site_path"))
+  tenant_id: str = Field(default=None, validation_alias=AliasChoices("tenantId","tenant_id"))
   recursive: bool = Field(default=False)
-  omit_files: bool = Field(default=False, validation_alias=AliasChoices("omitFiles","omit_files")),
-  omit_pages: bool = Field(default=False, validation_alias=AliasChoices("omitPages","omit_pages")),
-  omit_lists: bool = Field(default=False, validation_alias=AliasChoices("omitLists","omit_lists")),
   extension: list[str] = Field(default=None)
 class Sharepoint(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
@@ -41,10 +37,7 @@ class Sharepoint(IntegrationStrategy):
   def run(self) -> None:
     indexer_config = SharepointIndexerConfig(
       path=self.__data.site_path,
-      recursive=self.__data.recursive,
-      omit_files=self.__data.omit_files,
-      omit_pages=self.__data.omit_pages,
-      omit_lists=self.__data.omit_lists
+      recursive=self.__data.recursive
     )
     downloader_config = SharepointDownloaderConfig(
       download_dir=self.working_directory
@@ -53,15 +46,15 @@ class Sharepoint(IntegrationStrategy):
       access_config=SharepointAccessConfig(client_cred=self.__data.client_secret),
       client_id=self.__data.client_id,
       site=self.__data.site_url,
-      permissions_config=None
+      tenant= self.__data.tenant_id if self.__data.tenant_id else None
     )
     pipeline = self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
       connection_config,
       extension=self.__data.extension)
-    current_indexer_process = pipeline.indexer_step.process
-    pipeline.indexer_step.process = CustomSharepointIndexer(**vars(current_indexer_process))
+    #current_indexer_process = pipeline.indexer_step.process
+    #pipeline.indexer_step.process = CustomSharepointIndexer(**vars(current_indexer_process))
     pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)