unstructured-ingest 0.0.21__py3-none-any.whl → 0.0.23__py3-none-any.whl

unstructured_ingest/embed/voyageai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Optional
 
 import numpy as np
 from pydantic import Field, SecretStr
@@ -8,7 +8,7 @@ from unstructured_ingest.embed.interfaces import BaseEmbeddingEncoder, Embedding
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
-    from langchain_voyageai import VoyageAIEmbeddings
+    from voyageai import Client as VoyageAIClient
 
 
 class VoyageAIEmbeddingConfig(EmbeddingConfig):
@@ -16,28 +16,30 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
     embedder_model_name: str = Field(alias="model_name")
     batch_size: Optional[int] = Field(default=None)
     truncation: Optional[bool] = Field(default=None)
+    max_retries: int = 0
+    timeout_in_seconds: Optional[int] = None
 
     @requires_dependencies(
-        ["langchain", "langchain_voyageai"],
+        ["voyageai"],
         extras="embed-voyageai",
     )
-    def get_client(self) -> "VoyageAIEmbeddings":
-        """Creates a Langchain VoyageAI python client to embed elements."""
-        from langchain_voyageai import VoyageAIEmbeddings
-
-        return VoyageAIEmbeddings(
-            voyage_api_key=self.api_key,
-            model=self.embedder_model_name,
-            batch_size=self.batch_size,
-            truncation=self.truncation,
+    def get_client(self) -> "VoyageAIClient":
+        """Creates a VoyageAI python client to embed elements."""
+        from voyageai import Client as VoyageAIClient
+
+        client = VoyageAIClient(
+            api_key=self.api_key.get_secret_value(),
+            max_retries=self.max_retries,
+            timeout=self.timeout_in_seconds,
         )
+        return client
 
 
 @dataclass
 class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
     config: VoyageAIEmbeddingConfig
 
-    def get_exemplary_embedding(self) -> List[float]:
+    def get_exemplary_embedding(self) -> list[float]:
         return self.embed_query(query="A sample query.")
 
     @property
@@ -50,17 +52,20 @@ class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
         exemplary_embedding = self.get_exemplary_embedding()
         return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
 
-    def embed_documents(self, elements: List[dict]) -> List[dict]:
-        client = self.config.get_client()
-        embeddings = client.embed_documents([e.get("text", "") for e in elements])
+    def _embed_documents(self, elements: list[str]) -> list[list[float]]:
+        client: VoyageAIClient = self.config.get_client()
+        response = client.embed(texts=elements, model=self.config.embedder_model_name)
+        return response.embeddings
+
+    def embed_documents(self, elements: list[dict]) -> list[dict]:
+        embeddings = self._embed_documents([e.get("text", "") for e in elements])
         return self._add_embeddings_to_elements(elements, embeddings)
 
-    def embed_query(self, query: str) -> List[float]:
-        client = self.config.get_client()
-        return client.embed_query(query)
+    def embed_query(self, query: str) -> list[float]:
+        return self._embed_documents(elements=[query])[0]
 
     @staticmethod
-    def _add_embeddings_to_elements(elements, embeddings) -> List[dict]:
+    def _add_embeddings_to_elements(elements, embeddings) -> list[dict]:
         assert len(elements) == len(embeddings)
         elements_w_embedding = []
         for i, element in enumerate(elements):

unstructured_ingest/interfaces.py

@@ -204,14 +204,14 @@ class EmbeddingConfig(BaseConfig):
         if self.model_name:
             kwargs["model_name"] = self.model_name
         # TODO make this more dynamic to map to encoder configs
-        if self.provider == "langchain-openai":
+        if self.provider == "openai":
             from unstructured_ingest.embed.openai import (
                 OpenAIEmbeddingConfig,
                 OpenAIEmbeddingEncoder,
             )
 
             return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(**kwargs))
-        elif self.provider == "langchain-huggingface":
+        elif self.provider == "huggingface":
             from unstructured_ingest.embed.huggingface import (
                 HuggingFaceEmbeddingConfig,
                 HuggingFaceEmbeddingEncoder,
@@ -225,7 +225,7 @@ class EmbeddingConfig(BaseConfig):
             )
 
             return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(**kwargs))
-        elif self.provider == "langchain-aws-bedrock":
+        elif self.provider == "aws-bedrock":
             from unstructured_ingest.embed.bedrock import (
                 BedrockEmbeddingConfig,
                 BedrockEmbeddingEncoder,
@@ -238,14 +238,14 @@ class EmbeddingConfig(BaseConfig):
                     region_name=self.aws_region,
                 )
             )
-        elif self.provider == "langchain-vertexai":
+        elif self.provider == "vertexai":
             from unstructured_ingest.embed.vertexai import (
                 VertexAIEmbeddingConfig,
                 VertexAIEmbeddingEncoder,
             )
 
             return VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(**kwargs))
-        elif self.provider == "langchain-voyageai":
+        elif self.provider == "voyageai":
             from unstructured_ingest.embed.voyageai import (
                 VoyageAIEmbeddingConfig,
                 VoyageAIEmbeddingEncoder,

unstructured_ingest/v2/cli/base/cmd.py

@@ -155,7 +155,7 @@ class BaseCmd(ABC):
     @staticmethod
     def get_filterer(options: dict[str, Any]) -> Optional[Filterer]:
         filterer_configs = extract_config(flat_data=options, config=FiltererConfig)
-        if not filterer_configs.dict():
+        if not filterer_configs.model_dump():
             return None
         return Filterer(config=filterer_configs)
 

unstructured_ingest/v2/interfaces/connector.py

@@ -19,7 +19,7 @@ class ConnectionConfig(BaseModel):
     def get_access_config(self) -> dict[str, Any]:
         if not self.access_config:
             return {}
-        return self.access_config.get_secret_value().dict()
+        return self.access_config.get_secret_value().model_dump()
 
 
 ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)

unstructured_ingest/v2/pipeline/pipeline.py

@@ -187,7 +187,9 @@ class Pipeline:
         return filtered_records
 
     def _run(self):
-        logger.info(f"running local pipeline: {self} with configs: " f"{self.context.json()}")
+        logger.info(
+            f"running local pipeline: {self} with configs: " f"{self.context.model_dump_json()}"
+        )
         if self.context.mp_supported:
             manager = mp.Manager()
             self.context.status = manager.dict()

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -28,7 +28,7 @@ class ChunkStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.chunking_strategy})"
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     def should_chunk(self, filepath: Path, file_data: FileData) -> bool:

unstructured_ingest/v2/pipeline/steps/download.py

@@ -31,9 +31,13 @@ class DownloadStep(PipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = self.process.download_config.json() if self.process.download_config else None
+        config = (
+            self.process.download_config.model_dump_json() if self.process.download_config else None
+        )
         connection_config = (
-            self.process.connection_config.json() if self.process.connection_config else None
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
         )
         logger.info(
             f"Created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -28,7 +28,7 @@ class EmbedStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.embedding_provider})"
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     def should_embed(self, filepath: Path, file_data: FileData) -> bool:

unstructured_ingest/v2/pipeline/steps/filter.py

@@ -16,7 +16,7 @@ class FilterStep(PipelineStep):
     identifier: str = STEP_ID
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     async def _run_async(self, fn: Callable, file_data_path: str, **kwargs) -> Optional[dict]:

unstructured_ingest/v2/pipeline/steps/index.py

@@ -23,9 +23,11 @@ class IndexStep(PipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = self.process.index_config.json() if self.process.index_config else None
+        config = self.process.index_config.model_dump_json() if self.process.index_config else None
         connection_config = (
-            self.process.connection_config.json() if self.process.connection_config else None
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
         )
         logger.info(
             f"created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -28,7 +28,7 @@ class PartitionStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.strategy})"
 
     def __post_init__(self):
-        config = self.process.config.json()
+        config = self.process.config.model_dump_json()
         logger.info(f"created {self.identifier} with configs: {config}")
 
     def should_partition(self, filepath: Path, file_data: FileData) -> bool:

unstructured_ingest/v2/pipeline/steps/stage.py

@@ -28,7 +28,9 @@ class UploadStageStep(PipelineStep):
 
     def __post_init__(self):
         config = (
-            self.process.upload_stager_config.json() if self.process.upload_stager_config else None
+            self.process.upload_stager_config.model_dump_json()
+            if self.process.upload_stager_config
+            else None
         )
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         logger.info(f"created {self.identifier} with configs: {config}")

unstructured_ingest/v2/pipeline/steps/uncompress.py

@@ -22,7 +22,7 @@ class UncompressStep(PipelineStep):
     identifier: str = STEP_ID
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     async def _run_async(

unstructured_ingest/v2/pipeline/steps/upload.py

@@ -25,9 +25,13 @@ class UploadStep(BatchPipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = self.process.upload_config.json() if self.process.upload_config else None
+        config = (
+            self.process.upload_config.model_dump_json() if self.process.upload_config else None
+        )
         connection_config = (
-            self.process.connection_config.json() if self.process.connection_config else None
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
         )
         logger.info(
             f"Created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/processes/connectors/airtable.py

@@ -181,7 +181,7 @@ class AirtableIndexer(Indexer):
             yield FileData(
                 identifier=table_meta.get_id(),
                 connector_type=CONNECTOR_TYPE,
-                additional_metadata=table_meta.dict(),
+                additional_metadata=table_meta.model_dump(),
                 source_identifiers=SourceIdentifiers(
                     filename=str(Path(fullpath).name),
                     fullpath=fullpath,

unstructured_ingest/v2/processes/connectors/databricks_volumes.py

@@ -130,7 +130,7 @@ class DatabricksVolumesUploader(Uploader):
 
         return WorkspaceClient(
             host=self.connection_config.host,
-            **self.connection_config.access_config.get_secret_value().dict(),
+            **self.connection_config.access_config.get_secret_value().model_dump(),
         )
 
     def precheck(self) -> None:

unstructured_ingest/v2/processes/connectors/elasticsearch.py

@@ -104,8 +104,8 @@ class ElasticsearchConnectionConfig(ConnectionConfig):
         elif access_config.es_api_key:
             client_input_kwargs["api_key"] = access_config.es_api_key
         client_input = ElasticsearchClientInput(**client_input_kwargs)
-        logger.debug(f"elasticsearch client inputs mapped to: {client_input.dict()}")
-        client_kwargs = client_input.dict()
+        logger.debug(f"elasticsearch client inputs mapped to: {client_input.model_dump()}")
+        client_kwargs = client_input.model_dump()
         client_kwargs["basic_auth"] = (
             client_input.basic_auth.get_secret_value() if client_input.basic_auth else None
         )

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -84,7 +85,7 @@ class AzureConnectionConfig(FsspecConnectionConfig):
     def get_access_config(self) -> dict[str, Any]:
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs: dict[str, Any] = {
-            k: v for k, v in self.access_config.get_secret_value().dict().items() if v
+            k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v
         }
         return access_configs
 
@@ -99,14 +100,39 @@ class AzureIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
-    def sterilize_info(self, path) -> dict:
-        info = self.fs.info(path=path)
-        return sterilize_dict(data=info, default=azure_json_serial)
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data, default=azure_json_serial)
 
     @requires_dependencies(["adlfs", "fsspec"], extras="azure")
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = (
+            file_data.get("creation_time").timestamp() if "creation_time" in file_data else None
+        )
+        date_modified = (
+            file_data.get("last_modified").timestamp() if "last_modified" in file_data else None
+        )
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("etag")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class AzureDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -2,12 +2,14 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
+from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -52,7 +54,7 @@ class BoxConnectionConfig(FsspecConnectionConfig):
                 ac.box_app_config,
             ),
         }
-        access_config: dict[str, Any] = ac.dict()
+        access_config: dict[str, Any] = ac.model_dump()
         access_config.pop("box_app_config", None)
         access_kwargs_with_oauth.update(access_config)
 
@@ -73,6 +75,33 @@ class BoxIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = None
+        date_modified = None
+        if modified_at_str := file_data.get("modified_at"):
+            date_modified = parser.parse(modified_at_str).timestamp()
+        if created_at_str := file_data.get("created_at"):
+            date_created = parser.parse(created_at_str).timestamp()
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("id")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class BoxDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -22,7 +23,6 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_dict
 
 CONNECTOR_TYPE = "dropbox"
 
@@ -49,6 +49,40 @@ class DropboxIndexer(FsspecIndexer):
     index_config: DropboxIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"].lstrip("/")
+        date_created = None
+        date_modified = None
+        server_modified = file_data.get("server_modified")
+        client_modified = file_data.get("client_modified")
+        if server_modified and client_modified and server_modified > client_modified:
+            date_created = str(client_modified.timestamp())
+            date_modified = str(server_modified.timestamp())
+        elif server_modified and client_modified and server_modified < client_modified:
+            date_created = str(server_modified.timestamp())
+            date_modified = str(client_modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("content_hash")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def __post_init__(self):
         # dropbox expects the path to start with a /
@@ -63,12 +97,6 @@ class DropboxIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
-    def sterilize_info(self, path) -> dict:
-        # the fs.info method defined in the dropboxdrivefs library expects a "url"
-        # kwarg rather than "path"; though both refer to the same thing
-        info = self.fs.info(url=path)
-        return sterilize_dict(data=info)
-
 
 class DropboxDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -1,10 +1,7 @@
 from __future__ import annotations
 
-import contextlib
 from dataclasses import dataclass, field
-from datetime import datetime
 from pathlib import Path
-from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
@@ -113,18 +110,13 @@ class FsspecIndexer(Indexer):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
-    def list_files(self) -> list[str]:
+    def get_file_data(self) -> list[dict[str, Any]]:
         if not self.index_config.recursive:
             # fs.ls does not walk directories
             # directories that are listed in cloud storage can cause problems
             # because they are seen as 0 byte files
-            found = self.fs.ls(self.index_config.path_without_protocol, detail=True)
-            if isinstance(found, list):
-                return [
-                    x.get("name") for x in found if x.get("size") > 0 and x.get("type") == "file"
-                ]
-            else:
-                raise TypeError(f"unhandled response type from ls: {type(found)}")
+            files = self.fs.ls(self.index_config.path_without_protocol, detail=True)
+
         else:
             # fs.find will recursively walk directories
             # "size" is a common key for all the cloud protocols with fs
@@ -132,84 +124,40 @@ class FsspecIndexer(Indexer):
                 self.index_config.path_without_protocol,
                 detail=True,
             )
-            if isinstance(found, dict):
-                return [
-                    k for k, v in found.items() if v.get("size") > 0 and v.get("type") == "file"
-                ]
-            else:
-                raise TypeError(f"unhandled response type from find: {type(found)}")
-
-    def get_metadata(self, path: str) -> FileDataSourceMetadata:
-        date_created = None
-        date_modified = None
-        file_size = None
-        try:
-            created: Optional[Any] = self.fs.created(path)
-            if created:
-                if isinstance(created, datetime):
-                    date_created = str(created.timestamp())
-                else:
-                    date_created = str(created)
-        except NotImplementedError:
-            pass
+            files = found.values()
+        filtered_files = [
+            file for file in files if file.get("size") > 0 and file.get("type") == "file"
+        ]
+        return filtered_files
 
-        try:
-            modified: Optional[Any] = self.fs.modified(path)
-            if modified:
-                if isinstance(modified, datetime):
-                    date_modified = str(modified.timestamp())
-                else:
-                    date_modified = str(modified)
-        except NotImplementedError:
-            pass
-        with contextlib.suppress(AttributeError):
-            file_size = self.fs.size(path)
-
-        version = self.fs.checksum(path)
-        metadata: dict[str, str] = {}
-        with contextlib.suppress(AttributeError):
-            metadata = self.fs.metadata(path)
-        record_locator = {
-            "protocol": self.index_config.protocol,
-            "remote_file_path": self.index_config.remote_url,
-        }
-        file_stat = self.fs.stat(path=path)
-        if file_id := file_stat.get("id"):
-            record_locator["file_id"] = file_id
-        if metadata:
-            record_locator["metadata"] = metadata
-        return FileDataSourceMetadata(
-            date_created=date_created,
-            date_modified=date_modified,
-            date_processed=str(time()),
-            version=str(version),
-            url=f"{self.index_config.protocol}://{path}",
-            record_locator=record_locator,
-            filesize_bytes=file_size,
-        )
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        raise NotImplementedError()
+
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
 
-    def sterilize_info(self, path) -> dict:
-        info = self.fs.info(path=path)
-        return sterilize_dict(data=info)
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data)
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        files = self.list_files()
-        for file in files:
+        files = self.get_file_data()
+        for file_data in files:
+            file_path = self.get_path(file_data=file_data)
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
-            rel_path = file.replace(self.index_config.path_without_protocol, "").lstrip("/")
+            rel_path = file_path.replace(self.index_config.path_without_protocol, "").lstrip("/")
 
-            additional_metadata = self.sterilize_info(path=file)
-            additional_metadata["original_file_path"] = file
+            additional_metadata = self.sterilize_info(file_data=file_data)
+            additional_metadata["original_file_path"] = file_path
             yield FileData(
-                identifier=str(uuid5(NAMESPACE_DNS, file)),
+                identifier=str(uuid5(NAMESPACE_DNS, file_path)),
                 connector_type=self.connector_type,
                 source_identifiers=SourceIdentifiers(
-                    filename=Path(file).name,
+                    filename=Path(file_path).name,
                     rel_path=rel_path or None,
-                    fullpath=file,
+                    fullpath=file_path,
                 ),
-                metadata=self.get_metadata(path=file),
+                metadata=self.get_metadata(file_data=file_data),
                 additional_metadata=additional_metadata,
             )