unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -0,0 +1,185 @@
+import contextlib
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
+from unstructured_ingest.v2.interfaces import (
+    FileDataSourceMetadata,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+
+CONNECTOR_TYPE = "s3"
+
+if TYPE_CHECKING:
+    from s3fs import S3FileSystem
+
+
+class S3IndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+class S3AccessConfig(FsspecAccessConfig):
+    key: Optional[str] = Field(
+        default=None,
+        description="If not anonymous, use this access key ID, if specified. Takes precedence "
+        "over `aws_access_key_id` in client_kwargs.",
+    )
+    secret: Optional[str] = Field(
+        default=None, description="If not anonymous, use this secret access key, if specified."
+    )
+    token: Optional[str] = Field(
+        default=None, description="If not anonymous, use this security token, if specified."
+    )
+
+
+class S3ConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["s3", "s3a"], init=False)
+    access_config: Secret[S3AccessConfig] = Field(default=S3AccessConfig(), validate_default=True)
+    endpoint_url: Optional[str] = Field(
+        default=None,
+        description="Use this endpoint_url, if specified. Needed for "
+        "connecting to non-AWS S3 buckets.",
+    )
+    anonymous: bool = Field(
+        default=False, description="Connect to s3 without local AWS credentials."
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_access_config(self) -> dict[str, Any]:
+        access_configs: dict[str, Any] = {"anon": self.anonymous}
+        if self.endpoint_url:
+            access_configs["endpoint_url"] = self.endpoint_url
+
+        # Avoid injecting None by filtering out k,v pairs where the value is None
+        access_configs.update(
+            {k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v}
+        )
+        return access_configs
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["S3FileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+    def wrap_error(self, e: Exception) -> Exception:
+        # s3fs maps botocore errors into python ones using mapping here:
+        # https://github.com/fsspec/s3fs/blob/main/s3fs/errors.py
+        if isinstance(e, PermissionError):
+            return UserAuthError(e)
+        if isinstance(e, FileNotFoundError):
+            return UserError(e)
+        if cause := getattr(e, "__cause__", None):
+            error_response = cause.response
+            error_meta = error_response["ResponseMetadata"]
+            http_code = error_meta["HTTPStatusCode"]
+            message = error_response["Error"].get("Message", str(e))
+            if 400 <= http_code < 500:
+                return UserError(message)
+            if http_code >= 500:
+                return ProviderError(message)
+        logger.error(f"unhandled exception from s3 ({type(e)}): {e}", exc_info=True)
+        return e
+
+
+@dataclass
+class S3Indexer(FsspecIndexer):
+    connection_config: S3ConnectionConfig
+    index_config: S3IndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def wrap_error(self, e: Exception) -> Exception:
+        return self.connection_config.wrap_error(e=e)
+
+    def get_path(self, file_data: dict) -> str:
+        return file_data["Key"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["Key"]
+        date_created = None
+        date_modified = None
+        modified = file_data.get("LastModified")
+        if modified:
+            date_created = str(modified.timestamp())
+            date_modified = str(modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_size or file_data.get("Size")
+
+        version = file_data.get("ETag").rstrip('"').lstrip('"') if "ETag" in file_data else None
+        metadata: dict[str, str] = {}
+        with contextlib.suppress(AttributeError):
+            with self.connection_config.get_client(protocol=self.index_config.protocol) as client:
+                metadata = client.metadata(path=path)
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        if metadata:
+            record_locator["metadata"] = metadata
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class S3DownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class S3Downloader(FsspecDownloader):
+    protocol: str = "s3"
+    connection_config: S3ConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[S3DownloaderConfig] = field(default_factory=S3DownloaderConfig)
+
+
+class S3UploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class S3Uploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: S3ConnectionConfig
+    upload_config: S3UploaderConfig = field(default=None)
+
+
+s3_source_entry = SourceRegistryEntry(
+    indexer=S3Indexer,
+    indexer_config=S3IndexerConfig,
+    downloader=S3Downloader,
+    downloader_config=S3DownloaderConfig,
+    connection_config=S3ConnectionConfig,
+)
+
+s3_destination_entry = DestinationRegistryEntry(
+    uploader=S3Uploader,
+    uploader_config=S3UploaderConfig,
+    connection_config=S3ConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import FileData, FileDataSourceMetadata
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+
+if TYPE_CHECKING:
+    from fsspec.implementations.sftp import SFTPFileSystem
+
+CONNECTOR_TYPE = "sftp"
+
+
+class SftpIndexerConfig(FsspecIndexerConfig):
+    def model_post_init(self, __context: Any) -> None:
+        super().model_post_init(__context)
+        _, ext = os.path.splitext(self.remote_url)
+        parsed_url = urlparse(self.remote_url)
+        if ext:
+            self.path_without_protocol = Path(parsed_url.path).parent.as_posix().lstrip("/")
+        else:
+            self.path_without_protocol = parsed_url.path.lstrip("/")
+
+
+class SftpAccessConfig(FsspecAccessConfig):
+    password: str = Field(description="Password for sftp connection")
+
+
+class SftpConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = Field(default_factory=lambda: ["sftp"], init=False)
+    access_config: Secret[SftpAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    username: str = Field(description="Username for sftp connection")
+    host: Optional[str] = Field(default=None, description="Hostname for sftp connection")
+    port: int = Field(default=22, description="Port for sftp connection")
+    look_for_keys: bool = Field(
+        default=False, description="Whether to search for private key files in ~/.ssh/"
+    )
+    allow_agent: bool = Field(default=False, description="Whether to connect to the SSH agent.")
+
+    def get_access_config(self) -> dict[str, Any]:
+        access_config = {
+            "username": self.username,
+            "host": self.host,
+            "port": self.port,
+            "look_for_keys": self.look_for_keys,
+            "allow_agent": self.allow_agent,
+            "password": self.access_config.get_secret_value().password,
+        }
+        return access_config
+
+    @contextmanager
+    @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
+    def get_client(self, protocol: str) -> Generator["SFTPFileSystem", None, None]:
+        # The paramiko.SSHClient() client that's opened by the SFTPFileSystem
+        # never gets closed so explicitly adding that as part of this context manager
+        from fsspec import get_filesystem_class
+
+        client: SFTPFileSystem = get_filesystem_class(protocol)(
+            **self.get_access_config(),
+        )
+        yield client
+        client.client.close()
+
+
+@dataclass
+class SftpIndexer(FsspecIndexer):
+    connection_config: SftpConnectionConfig
+    index_config: SftpIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def __post_init__(self):
+        parsed_url = urlparse(self.index_config.remote_url)
+        self.connection_config.host = parsed_url.hostname or self.connection_config.host
+        self.connection_config.port = parsed_url.port or self.connection_config.port
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        for file in super().run(**kwargs):
+            new_identifier = (
+                f"sftp://"
+                f"{self.connection_config.host}:"
+                f"{self.connection_config.port}/"
+                f"{file.identifier}"
+            )
+            file.identifier = new_identifier
+            yield file
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = str(file_data.get("time").timestamp()) if "time" in file_data else None
+        date_modified = str(file_data.get("mtime").timestamp()) if "mtime" in file_data else None
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class SftpDownloaderConfig(FsspecDownloaderConfig):
+    remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")
+
+
+@dataclass
+class SftpDownloader(FsspecDownloader):
+    protocol: str = "sftp"
+    connection_config: SftpConnectionConfig
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    download_config: Optional[SftpDownloaderConfig] = field(default_factory=SftpDownloaderConfig)
+
+    def __post_init__(self):
+        parsed_url = urlparse(self.download_config.remote_url)
+        self.connection_config.host = parsed_url.hostname or self.connection_config.host
+        self.connection_config.port = parsed_url.port or self.connection_config.port
+
+
+class SftpUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class SftpUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: SftpConnectionConfig
+    upload_config: SftpUploaderConfig = field(default=None)
+
+
+sftp_source_entry = SourceRegistryEntry(
+    indexer=SftpIndexer,
+    indexer_config=SftpIndexerConfig,
+    downloader=SftpDownloader,
+    downloader_config=SftpDownloaderConfig,
+    connection_config=SftpConnectionConfig,
+)
+
+sftp_destination_entry = DestinationRegistryEntry(
+    uploader=SftpUploader,
+    uploader_config=SftpUploaderConfig,
+    connection_config=SftpConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/utils.py

@@ -0,0 +1,17 @@
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Callable
+
+
+def json_serial(obj):
+    if isinstance(obj, Path):
+        return obj.as_posix()
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializable" % type(obj))
+
+
+def sterilize_dict(data: dict, default: Callable = json_serial) -> dict:
+    data_s = json.dumps(data, default=default)
+    return json.loads(data_s)

unstructured_ingest/v2/processes/connectors/gitlab.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+
+from pydantic import Field, Secret, model_validator
+
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    FileData,
+    FileDataSourceMetadata,
+    Indexer,
+    IndexerConfig,
+    SourceIdentifiers,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+CONNECTOR_TYPE = "gitlab"
+if TYPE_CHECKING:
+    from gitlab import Gitlab
+    from gitlab.v4.objects.projects import Project
+
+
+class GitLabAccessConfig(AccessConfig):
+    access_token: Optional[str] = Field(
+        default=None,
+        description="Optional personal access token for authenticating with the GitLab API.",
+    )
+
+
+class GitLabConnectionConfig(ConnectionConfig):
+    access_config: Secret[GitLabAccessConfig] = Field(
+        default_factory=GitLabAccessConfig,
+        validate_default=True,
+        description="Secret configuration for accessing the GitLab API by authentication token.",
+    )
+    url: str = Field(description="The full URL to the GitLab project or repository.")
+    base_url: str = Field(
+        default="https://gitlab.com",
+        description="The base URL for the GitLab instance (default is GitLab's public domain).",
+    )
+    repo_path: str = Field(
+        default=None,
+        init=False,
+        repr=False,
+        description="The normalized path extracted from the repository URL.",
+    )
+
+    @model_validator(mode="after")
+    def set_repo_path(self):
+        """
+        Parses the provided GitLab URL to extract the `base_url` and `repo_path`,
+        ensuring both are properly formatted for use.
+
+        If the URL contains a scheme (e.g., 'https') and a network location (e.g., 'gitlab.com'),
+        the `base_url` is set accordingly. The repository path is extracted and normalized
+        by removing any leading slashes.
+
+        Notes:
+            - If the URL contains both a scheme and network location, the `base_url` is
+              extracted directly from the URL.
+            - The `repo_path` is adjusted to remove any leading slashes.
+            - This method assumes that the URL follows GitLab's structure
+              (e.g., 'https://gitlab.com/owner/repo').
+        """
+        parsed_gh_url = urlparse(self.url)
+
+        if parsed_gh_url.scheme and parsed_gh_url.netloc:
+            self.base_url = f"{parsed_gh_url.scheme}://{parsed_gh_url.netloc}"
+        self.repo_path = parsed_gh_url.path.lstrip("/")
+
+        return self
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["gitlab"], extras="gitlab")
+    @contextmanager
+    def get_client(self) -> Generator["Gitlab", None, None]:
+        from gitlab import Gitlab
+
+        logger.info(f"Connection to GitLab: {self.base_url!r}")
+        with Gitlab(
+            self.base_url, private_token=self.access_config.get_secret_value().access_token
+        ) as client:
+            yield client
+
+    @contextmanager
+    def get_project(self) -> Generator["Project", None, None]:
+        """Retrieves the specified GitLab project using the configured base URL and access token.
+
+        Returns:
+            Project: A GitLab `Project` object representing the specified repository.
+
+        Raises:
+            SourceConnectionError: If the GitLab API connection fails.
+            gitlab.exceptions.GitlabGetError: If the project is not found.
+        """
+        with self.get_client() as client:
+            logger.info(f"Accessing Project: '{self.repo_path}'")
+            project = client.projects.get(self.repo_path)
+
+            logger.info(f"Successfully accessed project '{self.repo_path}'")
+            yield project
+
+
+class GitLabIndexerConfig(IndexerConfig):
+    path: Path = Field(
+        default="/", description=("Path to the location in the repository that will be processed.")
+    )
+    recursive: bool = Field(
+        default=True,
+        description=(
+            "Flag to control recursive operations when indexing. "
+            "If True, the indexer will traverse directories recursively."
+        ),
+    )
+    git_branch: Optional[str] = Field(
+        default=None,
+        description="The name of the branch to interact with.",
+    )
+
+
+@dataclass
+class GitLabIndexer(Indexer):
+    connection_config: GitLabConnectionConfig
+    index_config: GitLabIndexerConfig
+
+    def precheck(self) -> None:
+        """Validates the connection to the GitLab instance by authenticating or
+        accessing the project.
+
+        This method ensures that the GitLab credentials and configuration are correct by
+        either authenticating or attempting to fetch the specified project.
+
+        Raises:
+            SourceConnectionError: If the connection or authentication with GitLab fails.
+        """
+
+        try:
+            with self.connection_config.get_client() as client:
+                if self.connection_config.access_config.get_secret_value().access_token is not None:
+                    client.auth()
+                else:
+                    client.projects.get(self.connection_config.repo_path)
+
+        except Exception as e:
+            logger.error(f"Failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"Failed to validate connection: {e}")
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        """Iterates over the GitLab repository tree and yields file metadata as `FileData` objects.
+
+        This method fetches the repository tree for the specified branch and iterates
+        over its contents. For each file (blob), it generates a `FileData` object containing
+        the file's metadata, path, and permissions.
+
+        Args:
+            **kwargs (Any): Additional keyword arguments (if required).
+
+        Yields:
+            FileData: A generator that yields `FileData` objects representing each file (blob)
+            in the repository.
+        """
+        with self.connection_config.get_project() as project:
+            ref = self.index_config.git_branch or project.default_branch
+
+            files = project.repository_tree(
+                path=str(self.index_config.path),
+                ref=ref,
+                recursive=self.index_config.recursive,
+                iterator=True,
+                all=True,
+            )
+
+        for file in files:
+            relative_path = str(Path(file["path"]).relative_to(self.index_config.path))
+            if file["type"] == "blob":
+                record_locator = {
+                    "file_path": file["path"],
+                    "ref": ref,
+                }
+
+                yield FileData(
+                    identifier=file["id"],
+                    connector_type=CONNECTOR_TYPE,
+                    source_identifiers=SourceIdentifiers(
+                        fullpath=file["path"],
+                        filename=Path(file["path"]).name,
+                        rel_path=relative_path,
+                    ),
+                    metadata=FileDataSourceMetadata(
+                        url=file["id"],
+                        record_locator=record_locator,
+                        permissions_data=[{"mode": file["mode"]}],
+                    ),
+                    additional_metadata={},
+                )
+
+
+class GitLabDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class GitLabDownloader(Downloader):
+    connection_config: GitLabConnectionConfig
+    download_config: GitLabDownloaderConfig
+
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        """Downloads a file from the repository and returns a `DownloadResponse`.
+
+        Args:
+            file_data (FileData): Metadata about the file to be downloaded.
+            **kwargs (Any): Additional arguments (if required).
+
+        Returns:
+            DownloadResponse: A response object containing the download details.
+        """
+        download_path = self.get_download_path(file_data=file_data)
+        if download_path is None:
+            logger.error(
+                "Generated download path is None, source_identifiers might be missing"
+                "from FileData."
+            )
+            raise ValueError("Generated invalid download path.")
+
+        self._download_file(file_data, download_path)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+    def _download_file(self, file_data: FileData, download_path: Path) -> None:
+        # NOTE: Indexer should supply the record locator in metadata
+        if (
+            file_data.metadata.record_locator is None
+            or "ref" not in file_data.metadata.record_locator
+            or "file_path" not in file_data.metadata.record_locator
+        ):
+            logger.error(
+                f"Invalid record locator in metadata: {file_data.metadata.record_locator}."
+                "Keys 'ref' and 'path' must be present."
+            )
+            raise ValueError("Invalid record locator.")
+
+        ref = file_data.metadata.record_locator["ref"]
+        path = file_data.metadata.record_locator["file_path"]
+        download_path.parent.mkdir(exist_ok=True, parents=True)
+
+        with self.connection_config.get_project() as project:
+            project_file = project.files.get(file_path=path, ref=ref)
+            with open(download_path, "wb") as file:
+                file.write(project_file.decode())
+
+
+gitlab_source_entry = SourceRegistryEntry(
+    connection_config=GitLabConnectionConfig,
+    indexer_config=GitLabIndexerConfig,
+    indexer=GitLabIndexer,
+    downloader_config=GitLabDownloaderConfig,
+    downloader=GitLabDownloader,
+)