unstructured-ingest 0.0.21__py3-none-any.whl → 0.0.22__py3-none-any.whl

unstructured_ingest/v2/pipeline/pipeline.py

@@ -187,7 +187,9 @@ class Pipeline:
         return filtered_records
 
     def _run(self):
-        logger.info(f"running local pipeline: {self} with configs: " f"{self.context.json()}")
+        logger.info(
+            f"running local pipeline: {self} with configs: " f"{self.context.model_dump_json()}"
+        )
         if self.context.mp_supported:
             manager = mp.Manager()
             self.context.status = manager.dict()

unstructured_ingest/v2/pipeline/steps/chunk.py

@@ -28,7 +28,7 @@ class ChunkStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.chunking_strategy})"
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     def should_chunk(self, filepath: Path, file_data: FileData) -> bool:

unstructured_ingest/v2/pipeline/steps/download.py

@@ -31,9 +31,13 @@ class DownloadStep(PipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = self.process.download_config.json() if self.process.download_config else None
+        config = (
+            self.process.download_config.model_dump_json() if self.process.download_config else None
+        )
         connection_config = (
-            self.process.connection_config.json() if self.process.connection_config else None
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
         )
         logger.info(
             f"Created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -28,7 +28,7 @@ class EmbedStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.embedding_provider})"
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     def should_embed(self, filepath: Path, file_data: FileData) -> bool:

unstructured_ingest/v2/pipeline/steps/filter.py

@@ -16,7 +16,7 @@ class FilterStep(PipelineStep):
     identifier: str = STEP_ID
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     async def _run_async(self, fn: Callable, file_data_path: str, **kwargs) -> Optional[dict]:

unstructured_ingest/v2/pipeline/steps/index.py

@@ -23,9 +23,11 @@ class IndexStep(PipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = self.process.index_config.json() if self.process.index_config else None
+        config = self.process.index_config.model_dump_json() if self.process.index_config else None
         connection_config = (
-            self.process.connection_config.json() if self.process.connection_config else None
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
         )
         logger.info(
             f"created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -28,7 +28,7 @@ class PartitionStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.strategy})"
 
     def __post_init__(self):
-        config = self.process.config.json()
+        config = self.process.config.model_dump_json()
         logger.info(f"created {self.identifier} with configs: {config}")
 
     def should_partition(self, filepath: Path, file_data: FileData) -> bool:

unstructured_ingest/v2/pipeline/steps/stage.py

@@ -28,7 +28,9 @@ class UploadStageStep(PipelineStep):
 
     def __post_init__(self):
         config = (
-            self.process.upload_stager_config.json() if self.process.upload_stager_config else None
+            self.process.upload_stager_config.model_dump_json()
+            if self.process.upload_stager_config
+            else None
         )
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         logger.info(f"created {self.identifier} with configs: {config}")

unstructured_ingest/v2/pipeline/steps/uncompress.py

@@ -22,7 +22,7 @@ class UncompressStep(PipelineStep):
     identifier: str = STEP_ID
 
     def __post_init__(self):
-        config = self.process.config.json() if self.process.config else None
+        config = self.process.config.model_dump_json() if self.process.config else None
         logger.info(f"created {self.identifier} with configs: {config}")
 
     async def _run_async(

unstructured_ingest/v2/pipeline/steps/upload.py

@@ -25,9 +25,13 @@ class UploadStep(BatchPipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = self.process.upload_config.json() if self.process.upload_config else None
+        config = (
+            self.process.upload_config.model_dump_json() if self.process.upload_config else None
+        )
         connection_config = (
-            self.process.connection_config.json() if self.process.connection_config else None
+            self.process.connection_config.model_dump_json()
+            if self.process.connection_config
+            else None
         )
         logger.info(
             f"Created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/processes/connectors/airtable.py

@@ -181,7 +181,7 @@ class AirtableIndexer(Indexer):
             yield FileData(
                 identifier=table_meta.get_id(),
                 connector_type=CONNECTOR_TYPE,
-                additional_metadata=table_meta.dict(),
+                additional_metadata=table_meta.model_dump(),
                 source_identifiers=SourceIdentifiers(
                     filename=str(Path(fullpath).name),
                     fullpath=fullpath,

unstructured_ingest/v2/processes/connectors/databricks_volumes.py

@@ -130,7 +130,7 @@ class DatabricksVolumesUploader(Uploader):
 
         return WorkspaceClient(
             host=self.connection_config.host,
-            **self.connection_config.access_config.get_secret_value().dict(),
+            **self.connection_config.access_config.get_secret_value().model_dump(),
         )
 
     def precheck(self) -> None:

unstructured_ingest/v2/processes/connectors/elasticsearch.py

@@ -104,8 +104,8 @@ class ElasticsearchConnectionConfig(ConnectionConfig):
         elif access_config.es_api_key:
             client_input_kwargs["api_key"] = access_config.es_api_key
         client_input = ElasticsearchClientInput(**client_input_kwargs)
-        logger.debug(f"elasticsearch client inputs mapped to: {client_input.dict()}")
-        client_kwargs = client_input.dict()
+        logger.debug(f"elasticsearch client inputs mapped to: {client_input.model_dump()}")
+        client_kwargs = client_input.model_dump()
         client_kwargs["basic_auth"] = (
             client_input.basic_auth.get_secret_value() if client_input.basic_auth else None
         )

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -84,7 +85,7 @@ class AzureConnectionConfig(FsspecConnectionConfig):
     def get_access_config(self) -> dict[str, Any]:
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs: dict[str, Any] = {
-            k: v for k, v in self.access_config.get_secret_value().dict().items() if v
+            k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v
         }
         return access_configs
 
@@ -99,14 +100,39 @@ class AzureIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
-    def sterilize_info(self, path) -> dict:
-        info = self.fs.info(path=path)
-        return sterilize_dict(data=info, default=azure_json_serial)
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data, default=azure_json_serial)
 
     @requires_dependencies(["adlfs", "fsspec"], extras="azure")
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = (
+            file_data.get("creation_time").timestamp() if "creation_time" in file_data else None
+        )
+        date_modified = (
+            file_data.get("last_modified").timestamp() if "last_modified" in file_data else None
+        )
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("etag")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class AzureDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -2,12 +2,14 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
+from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -52,7 +54,7 @@ class BoxConnectionConfig(FsspecConnectionConfig):
                 ac.box_app_config,
             ),
         }
-        access_config: dict[str, Any] = ac.dict()
+        access_config: dict[str, Any] = ac.model_dump()
         access_config.pop("box_app_config", None)
         access_kwargs_with_oauth.update(access_config)
 
@@ -73,6 +75,33 @@ class BoxIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = None
+        date_modified = None
+        if modified_at_str := file_data.get("modified_at"):
+            date_modified = parser.parse(modified_at_str).timestamp()
+        if created_at_str := file_data.get("created_at"):
+            date_created = parser.parse(created_at_str).timestamp()
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("id")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class BoxDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -22,7 +23,6 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_dict
 
 CONNECTOR_TYPE = "dropbox"
 
@@ -49,6 +49,40 @@ class DropboxIndexer(FsspecIndexer):
     index_config: DropboxIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"].lstrip("/")
+        date_created = None
+        date_modified = None
+        server_modified = file_data.get("server_modified")
+        client_modified = file_data.get("client_modified")
+        if server_modified and client_modified and server_modified > client_modified:
+            date_created = str(client_modified.timestamp())
+            date_modified = str(server_modified.timestamp())
+        elif server_modified and client_modified and server_modified < client_modified:
+            date_created = str(server_modified.timestamp())
+            date_modified = str(client_modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("content_hash")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def __post_init__(self):
         # dropbox expects the path to start with a /
@@ -63,12 +97,6 @@ class DropboxIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
-    def sterilize_info(self, path) -> dict:
-        # the fs.info method defined in the dropboxdrivefs library expects a "url"
-        # kwarg rather than "path"; though both refer to the same thing
-        info = self.fs.info(url=path)
-        return sterilize_dict(data=info)
-
 
 class DropboxDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -1,10 +1,7 @@
 from __future__ import annotations
 
-import contextlib
 from dataclasses import dataclass, field
-from datetime import datetime
 from pathlib import Path
-from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
@@ -113,18 +110,13 @@ class FsspecIndexer(Indexer):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
-    def list_files(self) -> list[str]:
+    def get_file_data(self) -> list[dict[str, Any]]:
         if not self.index_config.recursive:
             # fs.ls does not walk directories
             # directories that are listed in cloud storage can cause problems
             # because they are seen as 0 byte files
-            found = self.fs.ls(self.index_config.path_without_protocol, detail=True)
-            if isinstance(found, list):
-                return [
-                    x.get("name") for x in found if x.get("size") > 0 and x.get("type") == "file"
-                ]
-            else:
-                raise TypeError(f"unhandled response type from ls: {type(found)}")
+            files = self.fs.ls(self.index_config.path_without_protocol, detail=True)
+
         else:
             # fs.find will recursively walk directories
             # "size" is a common key for all the cloud protocols with fs
@@ -132,84 +124,40 @@ class FsspecIndexer(Indexer):
                 self.index_config.path_without_protocol,
                 detail=True,
             )
-            if isinstance(found, dict):
-                return [
-                    k for k, v in found.items() if v.get("size") > 0 and v.get("type") == "file"
-                ]
-            else:
-                raise TypeError(f"unhandled response type from find: {type(found)}")
-
-    def get_metadata(self, path: str) -> FileDataSourceMetadata:
-        date_created = None
-        date_modified = None
-        file_size = None
-        try:
-            created: Optional[Any] = self.fs.created(path)
-            if created:
-                if isinstance(created, datetime):
-                    date_created = str(created.timestamp())
-                else:
-                    date_created = str(created)
-        except NotImplementedError:
-            pass
+            files = found.values()
+        filtered_files = [
+            file for file in files if file.get("size") > 0 and file.get("type") == "file"
+        ]
+        return filtered_files
 
-        try:
-            modified: Optional[Any] = self.fs.modified(path)
-            if modified:
-                if isinstance(modified, datetime):
-                    date_modified = str(modified.timestamp())
-                else:
-                    date_modified = str(modified)
-        except NotImplementedError:
-            pass
-        with contextlib.suppress(AttributeError):
-            file_size = self.fs.size(path)
-
-        version = self.fs.checksum(path)
-        metadata: dict[str, str] = {}
-        with contextlib.suppress(AttributeError):
-            metadata = self.fs.metadata(path)
-        record_locator = {
-            "protocol": self.index_config.protocol,
-            "remote_file_path": self.index_config.remote_url,
-        }
-        file_stat = self.fs.stat(path=path)
-        if file_id := file_stat.get("id"):
-            record_locator["file_id"] = file_id
-        if metadata:
-            record_locator["metadata"] = metadata
-        return FileDataSourceMetadata(
-            date_created=date_created,
-            date_modified=date_modified,
-            date_processed=str(time()),
-            version=str(version),
-            url=f"{self.index_config.protocol}://{path}",
-            record_locator=record_locator,
-            filesize_bytes=file_size,
-        )
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        raise NotImplementedError()
+
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
 
-    def sterilize_info(self, path) -> dict:
-        info = self.fs.info(path=path)
-        return sterilize_dict(data=info)
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data)
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        files = self.list_files()
-        for file in files:
+        files = self.get_file_data()
+        for file_data in files:
+            file_path = self.get_path(file_data=file_data)
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
-            rel_path = file.replace(self.index_config.path_without_protocol, "").lstrip("/")
+            rel_path = file_path.replace(self.index_config.path_without_protocol, "").lstrip("/")
 
-            additional_metadata = self.sterilize_info(path=file)
-            additional_metadata["original_file_path"] = file
+            additional_metadata = self.sterilize_info(file_data=file_data)
+            additional_metadata["original_file_path"] = file_path
             yield FileData(
-                identifier=str(uuid5(NAMESPACE_DNS, file)),
+                identifier=str(uuid5(NAMESPACE_DNS, file_path)),
                 connector_type=self.connector_type,
                 source_identifiers=SourceIdentifiers(
-                    filename=Path(file).name,
+                    filename=Path(file_path).name,
                     rel_path=rel_path or None,
-                    fullpath=file,
+                    fullpath=file_path,
                 ),
-                metadata=self.get_metadata(path=file),
+                metadata=self.get_metadata(file_data=file_data),
                 additional_metadata=additional_metadata,
             )
 

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -2,13 +2,15 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional, Union
 
+from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import json_to_dict
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -106,6 +108,33 @@ class GcsIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = None
+        date_modified = None
+        if modified_at_str := file_data.get("updated"):
+            date_modified = parser.parse(modified_at_str).timestamp()
+        if created_at_str := file_data.get("timeCreated"):
+            date_created = parser.parse(created_at_str).timestamp()
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("etag")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class GcsDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -1,6 +1,5 @@
 import contextlib
 from dataclasses import dataclass, field
-from datetime import datetime
 from pathlib import Path
 from time import time
 from typing import Any, Generator, Optional
@@ -69,7 +68,7 @@ class S3ConnectionConfig(FsspecConnectionConfig):
 
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs.update(
-            {k: v for k, v in self.access_config.get_secret_value().dict().items() if v}
+            {k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v}
         )
         return access_configs
 
@@ -80,27 +79,25 @@ class S3Indexer(FsspecIndexer):
     index_config: S3IndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
-    def get_metadata(self, path: str) -> FileDataSourceMetadata:
+    def get_path(self, file_data: dict) -> str:
+        return file_data["Key"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["Key"]
         date_created = None
         date_modified = None
-        file_size = None
-        try:
-            modified: Optional[datetime] = self.fs.modified(path)
-            if modified:
-                date_created = str(modified.timestamp())
-                date_modified = str(modified.timestamp())
-        except NotImplementedError:
-            pass
-        with contextlib.suppress(AttributeError):
-            file_size = self.fs.size(path)
+        modified = file_data.get("LastModified")
+        if modified:
+            date_created = str(modified.timestamp())
+            date_modified = str(modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_size or file_data.get("Size")
 
-        version = None
-        info: dict[str, Any] = self.fs.info(path)
-        if etag := info.get("ETag"):
-            version = str(etag).rstrip('"').lstrip('"')
+        version = file_data.get("ETag").rstrip('"').lstrip('"') if "ETag" in file_data else None
         metadata: dict[str, str] = {}
         with contextlib.suppress(AttributeError):
-            metadata = self.fs.metadata(path)
+            metadata = self.fs.metadata(path=path)
         record_locator = {
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 from urllib.parse import urlparse
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -96,6 +97,26 @@ class SftpIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = file_data.get("time").timestamp() if "time" in file_data else None
+        date_modified = file_data.get("mtime").timestamp() if "mtime" in file_data else None
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class SftpDownloaderConfig(FsspecDownloaderConfig):
     remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")