unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Generator, Optional, Union
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.string_and_date_utils import json_to_dict
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+
+CONNECTOR_TYPE = "gcs"
+
+
+@dataclass
+class GcsIndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+@dataclass
+class GcsAccessConfig(FsspecAccessConfig):
+    service_account_key: Optional[str] = None
+    token: Union[str, dict, None] = field(init=False, default=None)
+
+    def __post_init__(self):
+        ALLOWED_AUTH_VALUES = "google_default", "cache", "anon", "browser", "cloud"
+
+        # Case: null value
+        if not self.service_account_key:
+            return
+
+        # Case: one of auth constants
+        if self.service_account_key in ALLOWED_AUTH_VALUES:
+            self.token = self.service_account_key
+            return
+
+        # Case: token as json
+        if isinstance(json_to_dict(self.service_account_key), dict):
+            self.token = json_to_dict(self.service_account_key)
+            return
+
+        # Case: path to token
+        if Path(self.service_account_key).is_file():
+            self.token = self.service_account_key
+            return
+
+        raise ValueError("Invalid auth token value")
+
+
+@dataclass
+class GcsConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"])
+    access_config: GcsAccessConfig = enhanced_field(
+        sensitive=True, default_factory=lambda: GcsAccessConfig()
+    )
+    connector_type: str = CONNECTOR_TYPE
+
+
+@dataclass
+class GcsIndexer(FsspecIndexer):
+    connection_config: GcsConnectionConfig
+    index_config: GcsIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        return super().run(**kwargs)
+
+
+@dataclass
+class GcsDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class GcsDownloader(FsspecDownloader):
+    protocol: str = "gcs"
+    connection_config: GcsConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[GcsDownloaderConfig] = field(default_factory=GcsDownloaderConfig)
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        return super().run(file_data=file_data, **kwargs)
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        return await super().run_async(file_data=file_data, **kwargs)
+
+
+@dataclass
+class GcsUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class GcsUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: GcsConnectionConfig
+    upload_config: GcsUploaderConfig = field(default=None)
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def __post_init__(self):
+        super().__post_init__()
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        return super().run(contents=contents, **kwargs)
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        return await super().run_async(path=path, file_data=file_data, **kwargs)
+
+
+gcs_source_entry = SourceRegistryEntry(
+    indexer=GcsIndexer,
+    indexer_config=GcsIndexerConfig,
+    downloader=GcsDownloader,
+    downloader_config=GcsDownloaderConfig,
+    connection_config=GcsConnectionConfig,
+)
+
+gcs_destination_entry = DestinationRegistryEntry(
+    uploader=GcsUploader,
+    uploader_config=GcsUploaderConfig,
+    connection_config=GcsConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -0,0 +1,164 @@
+import contextlib
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+from time import time
+from typing import Any, Generator, Optional
+
+from unstructured.documents.elements import DataSourceMetadata
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+
+CONNECTOR_TYPE = "s3"
+
+
+@dataclass
+class S3IndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+@dataclass
+class S3AccessConfig(FsspecAccessConfig):
+    key: Optional[str] = None
+    secret: Optional[str] = None
+    token: Optional[str] = None
+
+
+@dataclass
+class S3ConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["s3", "s3a"])
+    access_config: S3AccessConfig = enhanced_field(
+        sensitive=True, default_factory=lambda: S3AccessConfig()
+    )
+    endpoint_url: Optional[str] = None
+    anonymous: bool = False
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_access_config(self) -> dict[str, Any]:
+        access_configs: dict[str, Any] = {"anon": self.anonymous}
+        if self.endpoint_url:
+            access_configs["endpoint_url"] = self.endpoint_url
+
+        # Avoid injecting None by filtering out k,v pairs where the value is None
+        access_configs.update({k: v for k, v in self.access_config.to_dict().items() if v})
+        return access_configs
+
+
+@dataclass
+class S3Indexer(FsspecIndexer):
+    connection_config: S3ConnectionConfig
+    index_config: S3IndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_metadata(self, path: str) -> DataSourceMetadata:
+        date_created = None
+        date_modified = None
+        try:
+            modified: Optional[datetime] = self.fs.modified(path)
+            if modified:
+                date_created = str(modified.timestamp())
+                date_modified = str(modified.timestamp())
+        except NotImplementedError:
+            pass
+
+        version = None
+        info: dict[str, Any] = self.fs.info(path)
+        if etag := info.get("ETag"):
+            version = str(etag).rstrip('"').lstrip('"')
+        metadata: dict[str, str] = {}
+        with contextlib.suppress(AttributeError):
+            metadata = self.fs.metadata(path)
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        if metadata:
+            record_locator["metadata"] = metadata
+        return DataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+        )
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        return super().run(**kwargs)
+
+
+@dataclass
+class S3DownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class S3Downloader(FsspecDownloader):
+    protocol: str = "s3"
+    connection_config: S3ConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[S3DownloaderConfig] = field(default_factory=S3DownloaderConfig)
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        return super().run(file_data=file_data, **kwargs)
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        return await super().run_async(file_data=file_data, **kwargs)
+
+
+@dataclass
+class S3UploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class S3Uploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: S3ConnectionConfig
+    upload_config: S3UploaderConfig = field(default=None)
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def __post_init__(self):
+        super().__post_init__()
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        return super().run(contents=contents, **kwargs)
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        return await super().run_async(path=path, file_data=file_data, **kwargs)
+
+
+s3_source_entry = SourceRegistryEntry(
+    indexer=S3Indexer,
+    indexer_config=S3IndexerConfig,
+    downloader=S3Downloader,
+    downloader_config=S3DownloaderConfig,
+    connection_config=S3ConnectionConfig,
+)
+
+s3_destination_entry = DestinationRegistryEntry(
+    uploader=S3Uploader,
+    uploader_config=S3UploaderConfig,
+    connection_config=S3ConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Generator, Optional
+from urllib.parse import urlparse
+
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+
+CONNECTOR_TYPE = "sftp"
+
+
+@dataclass
+class SftpIndexerConfig(FsspecIndexerConfig):
+    def __post_init__(self):
+        super().__post_init__()
+        _, ext = os.path.splitext(self.remote_url)
+        parsed_url = urlparse(self.remote_url)
+        if ext:
+            self.path_without_protocol = Path(parsed_url.path).parent.as_posix().lstrip("/")
+        else:
+            self.path_without_protocol = parsed_url.path.lstrip("/")
+
+
+@dataclass
+class SftpAccessConfig(FsspecAccessConfig):
+    password: str
+
+
+@dataclass
+class SftpConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["sftp"])
+    access_config: SftpAccessConfig = enhanced_field(sensitive=True)
+    connector_type: str = CONNECTOR_TYPE
+    username: Optional[str] = None
+    host: Optional[str] = None
+    port: int = 22
+    look_for_keys: bool = False
+    allow_agent: bool = False
+
+    def get_access_config(self) -> dict[str, Any]:
+        access_config = {
+            "username": self.username,
+            "host": self.host,
+            "port": self.port,
+            "look_for_keys": self.look_for_keys,
+            "allow_agent": self.allow_agent,
+            "password": self.access_config.password,
+        }
+        return access_config
+
+
+@dataclass
+class SftpIndexer(FsspecIndexer):
+    connection_config: SftpConnectionConfig
+    index_config: SftpIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def __post_init__(self):
+        parsed_url = urlparse(self.index_config.remote_url)
+        self.connection_config.host = parsed_url.hostname or self.connection_config.host
+        self.connection_config.port = parsed_url.port or self.connection_config.port
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for file in super().run(**kwargs):
+            new_identifier = (
+                f"sftp://"
+                f"{self.connection_config.host}:"
+                f"{self.connection_config.port}/"
+                f"{file.identifier}"
+            )
+            file.identifier = new_identifier
+            yield file
+
+
+@dataclass
+class SftpDownloaderConfig(FsspecDownloaderConfig):
+    remote_url: Optional[str] = None
+
+    def __post_init__(self):
+        # TODO once python3.9 no longer supported and kw_only is allowed in dataclasses, remove:
+        if not self.remote_url:
+            raise TypeError(
+                f"{self.__class__.__name__}.__init__() "
+                f"missing 1 required positional argument: 'remote_url'"
+            )
+
+
+@dataclass
+class SftpDownloader(FsspecDownloader):
+    protocol: str = "sftp"
+    connection_config: SftpConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[SftpDownloaderConfig] = field(default_factory=SftpDownloaderConfig)
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def __post_init__(self):
+        parsed_url = urlparse(self.download_config.remote_url)
+        self.connection_config.host = parsed_url.hostname or self.connection_config.host
+        self.connection_config.port = parsed_url.port or self.connection_config.port
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        return super().run(file_data=file_data, **kwargs)
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        return await super().run_async(file_data=file_data, **kwargs)
+
+
+@dataclass
+class SftpUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class SftpUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: SftpConnectionConfig
+    upload_config: SftpUploaderConfig = field(default=None)
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def __post_init__(self):
+        super().__post_init__()
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        return super().run(contents=contents, **kwargs)
+
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        return await super().run_async(path=path, file_data=file_data, **kwargs)
+
+
+sftp_source_entry = SourceRegistryEntry(
+    indexer=SftpIndexer,
+    indexer_config=SftpIndexerConfig,
+    downloader=SftpDownloader,
+    downloader_config=SftpDownloaderConfig,
+    connection_config=SftpConnectionConfig,
+)
+
+sftp_destination_entry = DestinationRegistryEntry(
+    uploader=SftpUploader,
+    uploader_config=SftpUploaderConfig,
+    connection_config=SftpConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/utils.py

@@ -0,0 +1,17 @@
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Callable
+
+
+def json_serial(obj):
+    if isinstance(obj, Path):
+        return obj.as_posix()
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializable" % type(obj))
+
+
+def sterilize_dict(data: dict, default: Callable = json_serial) -> dict:
+    data_s = json.dumps(data, default=default)
+    return json.loads(data_s)