unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/fsspec/s3.py

@@ -0,0 +1,253 @@
+import contextlib
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import (
+    FileDataSourceMetadata,
+)
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+CONNECTOR_TYPE = "s3"
+
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines-avoid-characters  # noqa
+CHARACTERS_TO_AVOID = ["\\", "{", "^", "}", "%", "`", "]", '"', ">", "[", "~", "<", "#", "|"]
+
+if TYPE_CHECKING:
+    from s3fs import S3FileSystem
+
+
+class S3IndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+class S3AccessConfig(FsspecAccessConfig):
+    key: Optional[str] = Field(
+        default=None,
+        description="If not anonymous, use this access key ID, if specified. Takes precedence "
+        "over `aws_access_key_id` in client_kwargs.",
+    )
+    secret: Optional[str] = Field(
+        default=None, description="If not anonymous, use this secret access key, if specified."
+    )
+    token: Optional[str] = Field(
+        default=None, description="If not anonymous, use this security token, if specified."
+    )
+
+
+class S3ConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["s3", "s3a"], init=False)
+    access_config: Secret[S3AccessConfig] = Field(default=S3AccessConfig(), validate_default=True)
+    endpoint_url: Optional[str] = Field(
+        default=None,
+        description="Use this endpoint_url, if specified. Needed for "
+        "connecting to non-AWS S3 buckets.",
+    )
+    anonymous: bool = Field(
+        default=False, description="Connect to s3 without local AWS credentials."
+    )
+    ambient_credentials: bool = Field(
+        default=False,
+        description="Explicitly allow using ambient AWS credentials from .aws folder, "
+        "environment variables, or IAM roles. Requires ALLOW_AMBIENT_CREDENTIALS_S3 environment "
+        "variable to also be set to 'true' (case insensitive) for security. When False (default), "
+        "only explicit credentials or anonymous access are allowed.",
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_access_config(self) -> dict[str, Any]:
+        access_config = self.access_config.get_secret_value()
+        has_explicit_credentials = bool(
+            access_config.key or access_config.secret or access_config.token
+        )
+
+        access_configs: dict[str, Any]
+
+        if has_explicit_credentials:
+            access_configs = {"anon": False}
+            # Avoid injecting None by filtering out k,v pairs where the value is None
+            access_configs.update(
+                {k: v for k, v in access_config.model_dump().items() if v is not None}
+            )
+        elif self.ambient_credentials:
+            if os.getenv("ALLOW_AMBIENT_CREDENTIALS_S3", "").lower() == "true":
+                logger.info(
+                    "Using ambient AWS credentials (environment variables, .aws folder, IAM roles)"
+                )
+                access_configs = {"anon": False}
+                # Don't pass explicit credentials, let s3fs/boto3 auto-detect
+            else:
+                # Field allows but environment doesn't - raise error for security
+                raise UserAuthError(
+                    "Ambient credentials requested (ambient_credentials=True) but "
+                    "ALLOW_AMBIENT_CREDENTIALS_S3 environment variable is not set to 'true'. "
+                )
+        elif self.anonymous:
+            access_configs = {"anon": True}
+        else:
+            # User set anonymous=False but provided no credentials and no ambient permission
+            raise UserAuthError(
+                "No authentication method specified. anonymous=False but no explicit credentials "
+                "provided and ambient_credentials=False."
+            )
+
+        if self.endpoint_url:
+            access_configs["endpoint_url"] = self.endpoint_url
+
+        # This allows s3fs to properly follow AWS region redirects
+        access_configs["cache_regions"] = True
+
+        return access_configs
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["S3FileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+    def wrap_error(self, e: Exception) -> Exception:
+        # s3fs maps botocore errors into python ones using mapping here:
+        # https://github.com/fsspec/s3fs/blob/main/s3fs/errors.py
+        if isinstance(e, PermissionError):
+            return UserAuthError(e)
+        if isinstance(e, FileNotFoundError):
+            return UserError(f"File not found: {e}")
+        if cause := getattr(e, "__cause__", None):
+            error_response = cause.response
+            error_meta = error_response["ResponseMetadata"]
+            http_code = error_meta["HTTPStatusCode"]
+            message = error_response["Error"].get("Message", str(e))
+            if 400 <= http_code < 500:
+                return UserError(message)
+            if http_code >= 500:
+                return ProviderError(message)
+        logger.error(
+            "Unhandled exception from S3 (type: %s, endpoint: %s): %s",
+            type(e).__name__,
+            self.endpoint_url or "default",
+            e,
+            exc_info=True,
+        )
+        return e
+
+
+@dataclass
+class S3Indexer(FsspecIndexer):
+    connection_config: S3ConnectionConfig
+    index_config: S3IndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def wrap_error(self, e: Exception) -> Exception:
+        return self.connection_config.wrap_error(e=e)
+
+    def get_path(self, file_info: dict) -> str:
+        return file_info["Key"]
+
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["Key"]
+
+        self.log_debug("Getting metadata for S3 object", context={"file_path": path})
+        self.log_file_operation("Getting metadata", file_path=path)
+
+        date_created = None
+        date_modified = None
+        modified = file_info.get("LastModified")
+        if modified:
+            date_created = str(modified.timestamp())
+            date_modified = str(modified.timestamp())
+
+        file_size = file_info.get("size") if "size" in file_info else None
+        file_size = file_size or file_info.get("Size")
+
+        version = file_info.get("ETag").rstrip('"').lstrip('"') if "ETag" in file_info else None
+        metadata: dict[str, str] = {}
+        with (
+            contextlib.suppress(AttributeError),
+            self.connection_config.get_client(protocol=self.index_config.protocol) as client,
+        ):
+            metadata = client.metadata(path=path)
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        if metadata:
+            record_locator["metadata"] = metadata
+        issue_characters = [char for char in CHARACTERS_TO_AVOID if char in path]
+        if issue_characters:
+            self.log_warning(
+                f"File path contains characters that can cause issues with S3: {issue_characters}",
+                context={"path": path, "problematic_characters": issue_characters},
+            )
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class S3DownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class S3Downloader(FsspecDownloader):
+    protocol: str = "s3"
+    connection_config: S3ConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[S3DownloaderConfig] = field(default_factory=S3DownloaderConfig)
+
+
+class S3UploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class S3Uploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: S3ConnectionConfig
+    upload_config: S3UploaderConfig = field(default=None)
+
+
+s3_source_entry = SourceRegistryEntry(
+    indexer=S3Indexer,
+    indexer_config=S3IndexerConfig,
+    downloader=S3Downloader,
+    downloader_config=S3DownloaderConfig,
+    connection_config=S3ConnectionConfig,
+)
+
+s3_destination_entry = DestinationRegistryEntry(
+    uploader=S3Uploader,
+    uploader_config=S3UploaderConfig,
+    connection_config=S3ConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
+)

unstructured_ingest/processes/connectors/fsspec/sftp.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import FileData, FileDataSourceMetadata
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from fsspec.implementations.sftp import SFTPFileSystem
+
+CONNECTOR_TYPE = "sftp"
+
+
+class SftpIndexerConfig(FsspecIndexerConfig):
+    def model_post_init(self, __context: Any) -> None:
+        super().model_post_init(__context)
+        _, ext = os.path.splitext(self.remote_url)
+        parsed_url = urlparse(self.remote_url)
+        if ext:
+            self.path_without_protocol = Path(parsed_url.path).parent.as_posix().lstrip("/")
+        else:
+            self.path_without_protocol = parsed_url.path.lstrip("/")
+
+
+class SftpAccessConfig(FsspecAccessConfig):
+    password: str = Field(description="Password for sftp connection")
+
+
+class SftpConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = Field(default_factory=lambda: ["sftp"], init=False)
+    access_config: Secret[SftpAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    username: str = Field(description="Username for sftp connection")
+    host: Optional[str] = Field(default=None, description="Hostname for sftp connection")
+    port: int = Field(default=22, description="Port for sftp connection")
+    look_for_keys: bool = Field(
+        default=False, description="Whether to search for private key files in ~/.ssh/"
+    )
+    allow_agent: bool = Field(default=False, description="Whether to connect to the SSH agent.")
+
+    def get_access_config(self) -> dict[str, Any]:
+        access_config = {
+            "username": self.username,
+            "host": self.host,
+            "port": self.port,
+            "look_for_keys": self.look_for_keys,
+            "allow_agent": self.allow_agent,
+            "password": self.access_config.get_secret_value().password,
+        }
+        return access_config
+
+    @contextmanager
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def get_client(self, protocol: str) -> Generator["SFTPFileSystem", None, None]:
+        # The paramiko.SSHClient() client that's opened by the SFTPFileSystem
+        # never gets closed so explicitly adding that as part of this context manager
+        from fsspec import get_filesystem_class
+
+        client: SFTPFileSystem = get_filesystem_class(protocol)(
+            **self.get_access_config(),
+        )
+        yield client
+        client.client.close()
+
+
+@dataclass
+class SftpIndexer(FsspecIndexer):
+    connection_config: SftpConnectionConfig
+    index_config: SftpIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def __post_init__(self):
+        parsed_url = urlparse(self.index_config.remote_url)
+        self.connection_config.host = parsed_url.hostname or self.connection_config.host
+        self.connection_config.port = parsed_url.port or self.connection_config.port
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for file in super().run(**kwargs):
+            new_identifier = (
+                f"sftp://"
+                f"{self.connection_config.host}:"
+                f"{self.connection_config.port}/"
+                f"{file.identifier}"
+            )
+            file.identifier = new_identifier
+            yield file
+
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
+        date_created = str(file_info.get("time").timestamp()) if "time" in file_info else None
+        date_modified = str(file_info.get("mtime").timestamp()) if "mtime" in file_info else None
+
+        file_size = file_info.get("size") if "size" in file_info else None
+
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class SftpDownloaderConfig(FsspecDownloaderConfig):
+    remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")
+
+
+@dataclass
+class SftpDownloader(FsspecDownloader):
+    protocol: str = "sftp"
+    connection_config: SftpConnectionConfig
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    download_config: Optional[SftpDownloaderConfig] = field(default_factory=SftpDownloaderConfig)
+
+    def __post_init__(self):
+        parsed_url = urlparse(self.download_config.remote_url)
+        self.connection_config.host = parsed_url.hostname or self.connection_config.host
+        self.connection_config.port = parsed_url.port or self.connection_config.port
+
+
+class SftpUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class SftpUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: SftpConnectionConfig
+    upload_config: SftpUploaderConfig = field(default=None)
+
+
+sftp_source_entry = SourceRegistryEntry(
+    indexer=SftpIndexer,
+    indexer_config=SftpIndexerConfig,
+    downloader=SftpDownloader,
+    downloader_config=SftpDownloaderConfig,
+    connection_config=SftpConnectionConfig,
+)
+
+sftp_destination_entry = DestinationRegistryEntry(
+    uploader=SftpUploader,
+    uploader_config=SftpUploaderConfig,
+    connection_config=SftpConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
+)

unstructured_ingest/processes/connectors/fsspec/utils.py

@@ -0,0 +1,17 @@
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Callable
+
+
+def json_serial(obj):
+    if isinstance(obj, Path):
+        return obj.as_posix()
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializable" % type(obj))
+
+
+def sterilize_dict(data: dict, default: Callable = json_serial) -> dict:
+    data_s = json.dumps(data, default=default)
+    return json.loads(data_s)

unstructured_ingest/processes/connectors/github.py

@@ -0,0 +1,226 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import Field, Secret, field_validator
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.error import (
+    ProviderError,
+    UnstructuredIngestError,
+    UserAuthError,
+    UserError,
+)
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    Indexer,
+    IndexerConfig,
+    download_responses,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from github import ContentFile, GitTreeElement, Repository
+    from github import Github as GithubClient
+    from github.GithubException import GithubException
+    from requests import HTTPError
+
+CONNECTOR_TYPE = "github"
+
+
+class GithubAccessConfig(AccessConfig):
+    access_token: str = Field(description="Github acess token")
+
+
+class GithubConnectionConfig(ConnectionConfig):
+    access_config: Secret[GithubAccessConfig]
+    url: str = Field(description="Github url or repository owner/name pair")
+
+    @field_validator("url", mode="after")
+    def conform_url(cls, value: str):
+        parsed_url = urlparse(value)
+        return parsed_url.path
+
+    def get_full_url(self):
+        return f"https://github.com/{self.url}"
+
+    @requires_dependencies(["github"], extras="github")
+    def get_client(self) -> "GithubClient":
+        from github import Github as GithubClient
+
+        return GithubClient(login_or_token=self.access_config.get_secret_value().access_token)
+
+    def get_repo(self) -> "Repository":
+        client = self.get_client()
+        return client.get_repo(self.url)
+
+    def wrap_github_exception(self, e: "GithubException") -> Exception:
+        data = e.data
+        status_code = e.status
+        message = data.get("message")
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {message}")
+        if 400 <= status_code < 500:
+            return UserError(message)
+        if status_code > 500:
+            return ProviderError(message)
+        logger.debug(f"unhandled github error: {e}")
+        return e
+
+    def wrap_http_error(self, e: "HTTPError") -> Exception:
+        status_code = e.response.status_code
+        if status_code == 401:
+            return UserAuthError(f"Unauthorized access to Github: {e.response.text}")
+        if 400 <= status_code < 500:
+            return UserError(e.response.text)
+        if status_code > 500:
+            return ProviderError(e.response.text)
+        logger.debug(f"unhandled http error: {e}")
+        return UnstructuredIngestError(str(e))
+
+    @requires_dependencies(["requests"], extras="github")
+    def wrap_error(self, e: Exception) -> Exception:
+        from github.GithubException import GithubException
+        from requests import HTTPError
+
+        if isinstance(e, GithubException):
+            return self.wrap_github_exception(e=e)
+        if isinstance(e, HTTPError):
+            return self.wrap_http_error(e=e)
+        logger.debug(f"unhandled error: {e}")
+        return UnstructuredIngestError(str(e))
+
+
+class GithubIndexerConfig(IndexerConfig):
+    branch: Optional[str] = Field(
+        description="Branch to index, use the default if one isn't provided", default=None
+    )
+    recursive: bool = Field(
+        description="Recursively index all files in the repository", default=True
+    )
+
+
+@dataclass
+class GithubIndexer(Indexer):
+    connection_config: GithubConnectionConfig
+    index_config: GithubIndexerConfig = field(default_factory=GithubIndexerConfig)
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_repo()
+        except Exception as e:
+            raise self.connection_config.wrap_error(e=e)
+
+    def get_branch(self) -> str:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        return sha
+
+    def list_files(self) -> list["GitTreeElement"]:
+        repo = self.connection_config.get_repo()
+        sha = self.index_config.branch or repo.default_branch
+        git_tree = repo.get_git_tree(sha, recursive=self.index_config.recursive)
+        file_elements = [
+            element for element in git_tree.tree if element.size is not None and element.size > 0
+        ]
+        return file_elements
+
+    def convert_element(self, element: "GitTreeElement") -> FileData:
+        full_path = (
+            f"{self.connection_config.get_full_url()}/blob/{self.get_branch()}/{element.path}"
+        )
+
+        return FileData(
+            identifier=str(uuid5(NAMESPACE_DNS, full_path)),
+            connector_type=self.connector_type,
+            display_name=full_path,
+            source_identifiers=SourceIdentifiers(
+                filename=Path(element.path).name,
+                fullpath=(Path(self.get_branch()) / element.path).as_posix(),
+                rel_path=element.path,
+            ),
+            metadata=FileDataSourceMetadata(
+                url=element.url,
+                version=element.etag,
+                record_locator={},
+                date_modified=str(element.last_modified_datetime.timestamp()),
+                date_processed=str(time()),
+                filesize_bytes=element.size,
+                permissions_data=[{"mode": element.mode}],
+            ),
+        )
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for element in self.list_files():
+            yield self.convert_element(element=element)
+
+
+class GithubDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class GithubDownloader(Downloader):
+    download_config: GithubDownloaderConfig
+    connection_config: GithubConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    @requires_dependencies(["github"], extras="github")
+    def get_file(self, file_data: FileData) -> "ContentFile":
+        from github.GithubException import UnknownObjectException
+
+        path = file_data.source_identifiers.relative_path
+        repo = self.connection_config.get_repo()
+
+        try:
+            content_file = repo.get_contents(path)
+        except UnknownObjectException as e:
+            logger.error(f"File doesn't exists {self.connection_config.url}/{path}: {e}")
+            raise UserError(f"File not found: {path}")
+        return content_file
+
+    @requires_dependencies(["requests"], extras="github")
+    def get_contents(self, content_file: "ContentFile") -> bytes:
+        import requests
+
+        if content_file.decoded_content:
+            return content_file.decoded_content
+        download_url = content_file.download_url
+        resp = requests.get(download_url)
+        try:
+            resp.raise_for_status()
+        except requests.HTTPError as e:
+            raise self.connection_config.wrap_error(e=e)
+        return resp.content
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        content_file = self.get_file(file_data)
+        contents = self.get_contents(content_file)
+        download_path = self.get_download_path(file_data)
+        with download_path.open("wb") as f:
+            f.write(contents)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+github_source_entry = SourceRegistryEntry(
+    indexer=GithubIndexer,
+    indexer_config=GithubIndexerConfig,
+    downloader=GithubDownloader,
+    downloader_config=GithubDownloaderConfig,
+    connection_config=GithubConnectionConfig,
+)