ws-bom-robot-app 0.0.63__py3-none-any.whl → 0.0.103__py3-none-any.whl

ws_bom_robot_app/llm/vector_store/integration/api.py

@@ -0,0 +1,216 @@
+import asyncio, logging, aiohttp
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
+from langchain_core.documents import Document
+from ws_bom_robot_app.llm.vector_store.loader.base import Loader
+from typing import List, Union, Optional, Dict, Any, Literal
+from pydantic import BaseModel, Field, AliasChoices, field_validator
+import json
+import os
+
+
+class AuthConfig(BaseModel):
+  """
+  Configuration for API authentication.
+
+  Attributes:
+    type: Type of authentication (bearer, basic, api_key, custom, none)
+    token: Bearer token or API key value
+    username: Username for basic auth
+    password: Password for basic auth
+    header_name: Custom header name for API key
+    prefix: Prefix for the auth value (e.g., 'Bearer', 'Token')
+  """
+  type: Literal["bearer", "basic", "api_key", "custom", "none"] = Field(default="none")
+  token: Optional[str] = Field(default=None)
+  username: Optional[str] = Field(default=None)
+  password: Optional[str] = Field(default=None)
+  header_name: Optional[str] = Field(default=None, validation_alias=AliasChoices("headerName", "header_name"))
+  prefix: Optional[str] = Field(default=None)
+
+
+class ApiParams(BaseModel):
+  """
+  Generic API Integration Parameters.
+
+  Attributes:
+    url: The base URL of the API endpoint
+    method: HTTP method (GET, POST, PUT, DELETE, PATCH)
+    headers: Custom headers to include in the request
+    params: Query parameters for the request
+    body: Request body for POST/PUT/PATCH requests
+    auth: Authentication configuration
+    response_data_path: JSON path to extract data from response (e.g., 'data.items', 'results')
+    max_retries: Maximum number of retry attempts for failed requests
+    retry_delay: Base delay in seconds between retries (uses exponential backoff)
+    timeout: Request timeout in seconds
+  """
+  url: str = Field(validation_alias=AliasChoices("url", "endpoint"))
+  method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"] = Field(default="GET")
+  headers: Optional[Dict[str, str]] = Field(default_factory=dict)
+  params: Optional[Dict[str, Any]] = Field(default_factory=dict)
+  body: Optional[Union[Dict[str, Any], str]] = Field(default=None)
+  auth: Optional[AuthConfig] = Field(default_factory=lambda: AuthConfig())
+  response_data_path: Optional[str] = Field(default=None, validation_alias=AliasChoices("responseDataPath", "response_data_path"))
+  max_retries: int = Field(default=5, validation_alias=AliasChoices("maxRetries", "max_retries"))
+  retry_delay: float = Field(default=1.0, validation_alias=AliasChoices("retryDelay", "retry_delay"))
+  timeout: int = Field(default=30)
+
+  @field_validator('auth', mode='before')
+  @classmethod
+  def parse_auth(cls, v):
+    """Parse auth config from dict if needed"""
+    if isinstance(v, dict):
+      return AuthConfig(**v)
+    return v or AuthConfig()
+
+
+class Api(IntegrationStrategy):
+  """
+  Generic API Integration that supports:
+  - Multiple HTTP methods (GET, POST, PUT, DELETE, PATCH)
+  - Various authentication types (Bearer, Basic, API Key, Custom)
+  - Custom headers and parameters
+  - Automatic retry with exponential backoff
+  - Flexible response data extraction
+  """
+
+  def __init__(self, knowledgebase_path: str, data: dict[str, Union[str, int, list]]):
+    super().__init__(knowledgebase_path, data)
+    self.__data = ApiParams.model_validate(self.data)
+
+  def working_subdirectory(self) -> str:
+    return 'api_integration'
+
+  async def run(self) -> None:
+    """Fetch data from the API and save to JSON file"""
+    _data = await self.__fetch_data()
+    json_file_path = os.path.join(self.working_directory, 'api_data.json')
+    with open(json_file_path, 'w', encoding='utf-8') as f:
+      json.dump(_data, f, ensure_ascii=False, indent=2)
+    logging.info(f"Saved {len(_data) if isinstance(_data, list) else 1} items to {json_file_path}")
+
+  async def load(self) -> list[Document]:
+    """Load data from API and convert to documents"""
+    await self.run()
+    await asyncio.sleep(1)
+    return await Loader(self.working_directory).load()
+
+  def __prepare_headers(self) -> Dict[str, str]:
+    """Prepare request headers with authentication"""
+    headers = self.__data.headers.copy() if self.__data.headers else {}
+
+    # Add Content-Type if not present
+    if 'Content-Type' not in headers and self.__data.method in ["POST", "PUT", "PATCH"]:
+      headers['Content-Type'] = 'application/json'
+
+    # Add authentication
+    auth = self.__data.auth
+    if auth.type == "bearer":
+      prefix = auth.prefix or "Bearer"
+      headers['Authorization'] = f"{prefix} {auth.token}"
+    elif auth.type == "basic":
+      import base64
+      credentials = f"{auth.username}:{auth.password}"
+      encoded = base64.b64encode(credentials.encode()).decode()
+      headers['Authorization'] = f"Basic {encoded}"
+    elif auth.type == "api_key" and auth.header_name:
+      prefix = f"{auth.prefix} " if auth.prefix else ""
+      headers[auth.header_name] = f"{prefix}{auth.token}"
+
+    return headers
+
+  def __get_nested_value(self, data: Any, path: Optional[str]) -> Any:
+    """Extract nested value from data using dot notation path"""
+    if not path:
+      return data
+
+    keys = path.split('.')
+    current = data
+    for key in keys:
+      if isinstance(current, dict):
+        current = current.get(key)
+      elif isinstance(current, list) and key.isdigit():
+        current = current[int(key)]
+      else:
+        return None
+
+      if current is None:
+        return None
+
+    return current
+
+  async def __make_request(
+    self,
+    url: str,
+    headers: Dict[str, str],
+    params: Optional[Dict[str, Any]] = None
+  ) -> Dict[str, Any]:
+    """Make HTTP request with retry logic"""
+    retry_count = 0
+
+    while retry_count <= self.__data.max_retries:
+      try:
+        timeout = aiohttp.ClientTimeout(total=self.__data.timeout)
+
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+          request_kwargs = {
+            "headers": headers,
+            "params": params or self.__data.params
+          }
+
+          # Add body for POST/PUT/PATCH
+          if self.__data.method in ["POST", "PUT", "PATCH"] and self.__data.body:
+            if isinstance(self.__data.body, dict):
+              request_kwargs["json"] = self.__data.body
+            else:
+              request_kwargs["data"] = self.__data.body
+
+          async with session.request(
+            self.__data.method,
+            url,
+            **request_kwargs
+          ) as response:
+            # Check response status
+            if response.status == 429:  # Rate limit
+              retry_count += 1
+              if retry_count > self.__data.max_retries:
+                raise Exception("Rate limit exceeded. Maximum retries reached.")
+
+              wait_time = self.__data.retry_delay * (2 ** retry_count)
+              logging.warning(f"Rate limited. Waiting {wait_time}s (Attempt {retry_count}/{self.__data.max_retries})")
+              await asyncio.sleep(wait_time)
+              continue
+
+            response.raise_for_status()
+
+            # Parse response
+            try:
+              data = await response.json()
+              return data
+            except aiohttp.ContentTypeError:
+              text = await response.text()
+              logging.warning(f"Non-JSON response received: {text[:200]}")
+              return {"text": text}
+
+      except aiohttp.ClientError as e:
+        retry_count += 1
+        if retry_count > self.__data.max_retries:
+          raise Exception(f"Request failed after {self.__data.max_retries} retries: {e}")
+
+        wait_time = self.__data.retry_delay * (2 ** retry_count)
+        logging.warning(f"Request error: {e}. Retrying in {wait_time}s...")
+        await asyncio.sleep(wait_time)
+        continue
+
+    raise Exception("Maximum retries exceeded")
+
+  async def __fetch_data(self) -> Any:
+    """Fetch data from API"""
+    headers = self.__prepare_headers()
+    response = await self.__make_request(self.__data.url, headers)
+
+    # Extract data from response using path if specified
+    data = self.__get_nested_value(response, self.__data.response_data_path)
+    result = data if data is not None else response
+
+    return result

ws_bom_robot_app/llm/vector_store/integration/azure.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.azure import AzureConnectionConfig, AzureAccessConfig, AzureDownloaderConfig, AzureIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/base.py

@@ -1,10 +1,17 @@
-import os
+import os, copy
+from random import random
 from langchain_core.documents import Document
 from abc import ABC, abstractmethod
-from unstructured_ingest.v2.interfaces import ProcessorConfig
-from unstructured_ingest.v2.pipeline.pipeline import Pipeline, PartitionerConfig, FiltererConfig
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.pipeline.pipeline import (
+  Pipeline,
+  PartitionerConfig,
+  FiltererConfig
+)
+from unstructured_ingest.processes.connector_registry import source_registry
 from typing import Union
 from ws_bom_robot_app.llm.utils.secrets import Secrets
+from ws_bom_robot_app.config import config
 
 class IntegrationStrategy(ABC):
   @classmethod
@@ -32,23 +39,59 @@ class IntegrationStrategy(ABC):
     pass
 
 class UnstructuredIngest():
+  _PIPELINE: Pipeline = None
   def __init__(self, working_directory: str):
     self.working_directory = working_directory
-  def pipeline(self,indexer,downloader,connection,extension: list[str] = None) -> Pipeline:
-    return Pipeline.from_configs(
-      context=ProcessorConfig(
+    self._runtime_options = config.runtime_options()
+  def pipeline(self,indexer_config,downloader_config,connection_config,extension: list[str] = None) -> Pipeline:
+    def _default_processor_config() -> ProcessorConfig:
+      return ProcessorConfig(
         reprocess=False,
         verbose=False,
         tqdm=False,
-        num_processes=2,
+        num_processes=config.robot_ingest_max_threads, #safe choice to 1, avoid potential process-related issues with Docker
+        disable_parallelism=False,
         preserve_downloads=True,
         download_only=True,
-        raise_on_error=False
-      ),
-      indexer_config=indexer,
-      downloader_config=downloader,
-      source_connection_config=connection,
-      partitioner_config=PartitionerConfig(),
-      filterer_config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)
-    )
+        raise_on_error=False,
+        iter_delete=True,
+        delete_cache=False #already managed by the generator task
+      )
+    def _init_pipeline() -> Pipeline:
+      return Pipeline.from_configs(
+        context=_default_processor_config(),
+        indexer_config=indexer_config,
+        downloader_config=downloader_config,
+        source_connection_config=connection_config,
+        partitioner_config=PartitionerConfig(),
+        filterer_config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)
+      )
+    def _instance_pipeline() -> Pipeline:
+        from unstructured_ingest.pipeline.steps.index import  IndexStep
+        from unstructured_ingest.pipeline.steps.download import  DownloadStep
+        from unstructured_ingest.pipeline.steps.filter import Filterer, FilterStep
+        _context = _default_processor_config()
+        source_entry = {
+                    k: v
+                    for k, v in source_registry.items()
+                    if type(indexer_config) is v.indexer_config
+                    and type(downloader_config) is v.downloader_config
+                    and type(connection_config) is v.connection_config
+                }
+        source = list(source_entry.values())[0]
+        _pipeline = copy.deepcopy(UnstructuredIngest._PIPELINE)
+        _pipeline.context = _context
+        _pipeline.context.work_dir = f"{self.working_directory}_unstructured" # use sibling directory, cleaned up by the generator task
+        _pipeline.indexer_step = IndexStep(process=source.indexer(index_config=indexer_config, connection_config=connection_config), context=_context)
+        _pipeline.downloader_step = DownloadStep(process=source.downloader(download_config=downloader_config, connection_config=connection_config), context=_context)
+        _pipeline.filter_step = FilterStep(process=Filterer(config=FiltererConfig(file_glob=[f"**/*{ext}" for ext in extension] if extension else None)), context=_context) if extension else None
+        return _pipeline
 
+    if not UnstructuredIngest._PIPELINE:
+      import random
+      import time
+      time.sleep(random.uniform(0.2, 1))
+      if not UnstructuredIngest._PIPELINE:
+        UnstructuredIngest._PIPELINE = _init_pipeline()
+
+    return _instance_pipeline()

ws_bom_robot_app/llm/vector_store/integration/confluence.py

@@ -1,9 +1,10 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.processes.connectors.confluence import ConfluenceIndexerConfig, ConfluenceIndexer, ConfluenceDownloaderConfig, ConfluenceConnectionConfig, ConfluenceAccessConfig
+from unstructured_ingest.pipeline.pipeline import Pipeline
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
-from typing import Optional, Union
+from typing import List, Optional, Union
 from pydantic import BaseModel, Field, AliasChoices
 
 class ConfluenceParams(BaseModel):
@@ -16,6 +17,7 @@ class ConfluenceParams(BaseModel):
     password: Confluence password or Cloud API token, if filled, set the access_token to None and vice versa.
     access_token (str): The personal access token for authenticating with Confluence, e.g., 'AT....'
     spaces (list[str]): A list of Confluence spaces to interact with, e.g., ['SPACE1', 'SPACE2'].
+    max_num_of_docs_from_each_space (int): The maximum number of documents to fetch from each space. Defaults to 500, with a maximum limit of 5000.
     extension (list[str], optional): A list of file extensions to filter by. Defaults to None, e.g., ['.pdf', '.docx'].
   """
   url: str
@@ -23,6 +25,7 @@ class ConfluenceParams(BaseModel):
   password: Optional[str] = None
   access_token: Optional[str] = Field(None, validation_alias=AliasChoices("accessToken","access_token"))
   spaces: list[str] = []
+  max_num_of_docs_from_each_space: int = Field(default=500, ge=1, le=5000,validation_alias=AliasChoices("maxNumOfDocsFromEachSpace","max_num_of_docs_from_each_space"))
   extension: list[str] = Field(default=None)
 class Confluence(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
@@ -33,7 +36,8 @@ class Confluence(IntegrationStrategy):
     return 'confluence'
   def run(self) -> None:
     indexer_config = ConfluenceIndexerConfig(
-      spaces=self.__data.spaces
+      spaces=self.__data.spaces,
+      max_num_of_docs_from_each_space=self.__data.max_num_of_docs_from_each_space
     )
     downloader_config = ConfluenceDownloaderConfig(
       download_dir=self.working_directory
@@ -43,13 +47,37 @@ class Confluence(IntegrationStrategy):
       url=self.__data.url,
       username=self.__data.username
     )
-    self.__unstructured_ingest.pipeline(
+    pipeline: Pipeline = self.__unstructured_ingest.pipeline(
       indexer_config,
       downloader_config,
       connection_config,
-      extension=self.__data.extension).run()
+      extension=self.__data.extension
+    )
+    pipeline.indexer_step.process = CustomConfluenceIndexer(**vars(pipeline.indexer_step.process))
+    pipeline.run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)
       return await Loader(self.working_directory).load()
 
+class CustomConfluenceIndexer(ConfluenceIndexer):
+  def __init__(self, **kwargs):
+    for key, value in kwargs.items():
+        try:
+            setattr(super(), key, value)
+        except AttributeError:
+            setattr(self, key, value)
+  def _get_docs_ids_within_one_space(self, space_key: str) -> List[dict]:
+      with self.connection_config.get_client() as client:
+          pages = client.get_all_pages_from_space(
+              space=space_key,
+              start=0,
+              limit=self.index_config.max_num_of_docs_from_each_space, #explicitly limit the number of pages fetched (omitted  in unstructured-ingest)
+              expand=None,
+              content_type="page",  # blogpost and comment types not currently supported
+              status=None,
+          )
+      limited_pages = pages[: self.index_config.max_num_of_docs_from_each_space]
+      doc_ids = [{"space_id": space_key, "doc_id": page["id"]} for page in limited_pages]
+      return doc_ids
+

ws_bom_robot_app/llm/vector_store/integration/dropbox.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
+from unstructured_ingest.processes.connectors.fsspec.dropbox import DropboxConnectionConfig, DropboxAccessConfig, DropboxDownloaderConfig, DropboxIndexerConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union

ws_bom_robot_app/llm/vector_store/integration/gcs.py

@@ -1,6 +1,6 @@
 import asyncio
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
+from unstructured_ingest.processes.connectors.fsspec.gcs import GcsIndexerConfig, GcsConnectionConfig, GcsAccessConfig, GcsDownloaderConfig
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union, Optional

ws_bom_robot_app/llm/vector_store/integration/github.py

@@ -1,10 +1,12 @@
 import asyncio
 from typing import Optional, Union
-from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy
-from unstructured_ingest.interfaces import  ProcessorConfig, ReadConfig
-from unstructured_ingest.connector.git import GitAccessConfig
-from unstructured_ingest.connector.github import SimpleGitHubConfig
-from unstructured_ingest.runner import GithubRunner
+from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
+from unstructured_ingest.processes.connectors.github import (
+    GithubIndexerConfig,
+    GithubDownloaderConfig,
+    GithubConnectionConfig,
+    GithubAccessConfig
+)
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from pydantic import BaseModel, Field, AliasChoices
@@ -27,28 +29,26 @@ class Github(IntegrationStrategy):
   def __init__(self, knowledgebase_path: str, data: dict[str, Union[str,int,list]]):
     super().__init__(knowledgebase_path, data)
     self.__data = GithubParams.model_validate(self.data)
+    self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
   def working_subdirectory(self) -> str:
     return 'github'
   def run(self) -> None:
-    access_config = GitAccessConfig(
-      access_token=self.__data.access_token
-    )
-    file_ext = self.__data.file_ext or None
-    file_glob = [f"**/*{ext}" for ext in file_ext] if file_ext else None
-    config = SimpleGitHubConfig(
-      url = self.__data.repo,
-      access_config=access_config,
+    indexer_config = GithubIndexerConfig(
       branch=self.__data.branch,
-      file_glob=file_glob
+      recursive=True
+    )
+    downloader_config = GithubDownloaderConfig(
+      download_dir=self.working_directory
+    )
+    connection_config = GithubConnectionConfig(
+      access_config=GithubAccessConfig(access_token=self.__data.access_token),
+      url=self.__data.repo
     )
-    runner = GithubRunner(
-      connector_config=config,
-      processor_config=ProcessorConfig(reprocess=False,verbose=False,num_processes=2,raise_on_error=False),
-      read_config=ReadConfig(download_dir=self.working_directory,re_download=True,preserve_downloads=True,download_only=True),
-      partition_config=None,
-      retry_strategy_config=None
-      )
-    runner.run()
+    self.__unstructured_ingest.pipeline(
+      indexer_config,
+      downloader_config,
+      connection_config,
+      extension=self.__data.file_ext).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)

ws_bom_robot_app/llm/vector_store/integration/googledrive.py

@@ -1,10 +1,38 @@
 import asyncio
+import json
+from pathlib import Path
 from ws_bom_robot_app.llm.vector_store.integration.base import IntegrationStrategy, UnstructuredIngest
-from unstructured_ingest.v2.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.processes.connectors.google_drive import GoogleDriveConnectionConfig, GoogleDriveDownloaderConfig, GoogleDriveIndexerConfig, GoogleDriveAccessConfig
+from unstructured_ingest.data_types.file_data import FileData as OriginalFileData, BatchFileData as OriginalBatchFileData
 from langchain_core.documents import Document
 from ws_bom_robot_app.llm.vector_store.loader.base import Loader
 from typing import Union
 from pydantic import BaseModel, Field, AliasChoices
+
+# UTF-8 safe FileData classes
+class FileData(OriginalFileData):
+  @classmethod
+  def from_file(cls, path: str):
+    path = Path(path).resolve()
+    if not path.exists() or not path.is_file():
+      raise ValueError(f"file path not valid: {path}")
+    for encoding in ['utf-8', 'cp1252', 'iso-8859-1', 'latin-1']:
+      try:
+        with open(str(path), "r", encoding=encoding) as f:
+          return cls.model_validate(json.load(f))
+      except (UnicodeDecodeError, UnicodeError):
+        continue
+    raise ValueError(f"Could not decode file {path} with any supported encoding")
+
+  def to_file(self, path: str) -> None:
+    path = Path(path).resolve()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(str(path), "w", encoding="utf-8") as f:
+      json.dump(self.model_dump(), f, indent=2, ensure_ascii=False)
+
+class BatchFileData(OriginalBatchFileData, FileData):
+  pass
+
 class GoogleDriveParams(BaseModel):
   """
   GoogleDriveParams is a model that holds parameters for Google Drive integration.
@@ -42,26 +70,27 @@ class GoogleDrive(IntegrationStrategy):
     super().__init__(knowledgebase_path, data)
     self.__data = GoogleDriveParams.model_validate(self.data)
     self.__unstructured_ingest = UnstructuredIngest(self.working_directory)
+    self._apply_encoding_fix()
+
+  def _apply_encoding_fix(self):
+    """Replace FileData classes with UTF-8 safe versions"""
+    import unstructured_ingest.data_types.file_data as fd
+    fd.FileData = FileData
+    fd.BatchFileData = BatchFileData
+    fd.file_data_from_file = lambda path: BatchFileData.from_file(path) if path else FileData.from_file(path)
+
   def working_subdirectory(self) -> str:
     return 'googledrive'
+
   def run(self) -> None:
-    indexer_config = GoogleDriveIndexerConfig(
-      extensions=self.__data.extensions,
-      recursive=self.__data.recursive
-    )
-    downloader_config = GoogleDriveDownloaderConfig(
-      download_dir=self.working_directory
-    )
-    connection_config = GoogleDriveConnectionConfig(
-      access_config=GoogleDriveAccessConfig(
-        service_account_key=self.__data.service_account_key
-        ),
-      drive_id=self.__data.drive_id
-    )
     self.__unstructured_ingest.pipeline(
-      indexer_config,
-      downloader_config,
-      connection_config).run()
+      GoogleDriveIndexerConfig(extensions=self.__data.extensions, recursive=self.__data.recursive),
+      GoogleDriveDownloaderConfig(download_dir=self.working_directory),
+      GoogleDriveConnectionConfig(
+        access_config=GoogleDriveAccessConfig(service_account_key=self.__data.service_account_key),
+        drive_id=self.__data.drive_id
+      )
+    ).run()
   async def load(self) -> list[Document]:
       await asyncio.to_thread(self.run)
       await asyncio.sleep(1)