unstructured-ingest 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/couchbase.py

@@ -205,6 +205,7 @@ class CouchbaseIndexer(Indexer):
             yield FileData(
                 identifier=identified,
                 connector_type=CONNECTOR_TYPE,
+                doc_type="batch",
                 metadata=FileDataSourceMetadata(
                     url=f"{self.connection_config.connection_string}/"
                     f"{self.connection_config.bucket}",

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -148,9 +148,7 @@ class DatabricksVolumesDownloader(Downloader, ABC):
 
 
 class DatabricksVolumesUploaderConfig(UploaderConfig, DatabricksPathMixin):
-    overwrite: bool = Field(
-        default=False, description="If true, an existing file will be overwritten."
-    )
+    pass
 
 
 @dataclass
@@ -173,5 +171,5 @@ class DatabricksVolumesUploader(Uploader, ABC):
             self.connection_config.get_client().files.upload(
                 file_path=output_path,
                 contents=elements_file,
-                overwrite=self.upload_config.overwrite,
+                overwrite=True,
             )

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 from multiprocessing import Process
 from pathlib import Path
 from typing import Any, Optional
+from urllib.parse import urlparse
 
 import pandas as pd
 from pydantic import Field, Secret
@@ -94,7 +95,7 @@ class DeltaTableUploader(Uploader):
     connection_config: DeltaTableConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
-    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    @requires_dependencies(["boto3"], extras="delta-table")
     def precheck(self):
         secrets = self.connection_config.access_config.get_secret_value()
         if (
@@ -102,13 +103,24 @@ class DeltaTableUploader(Uploader):
             and secrets.aws_access_key_id
             and secrets.aws_secret_access_key
         ):
-            from fsspec import get_filesystem_class
+            from boto3 import client
+
+            url = urlparse(self.connection_config.table_uri)
+            bucket_name = url.netloc
+            dir_path = url.path.lstrip("/")
 
             try:
-                fs = get_filesystem_class("s3")(
-                    key=secrets.aws_access_key_id, secret=secrets.aws_secret_access_key
+                s3_client = client(
+                    "s3",
+                    aws_access_key_id=secrets.aws_access_key_id,
+                    aws_secret_access_key=secrets.aws_secret_access_key,
                 )
-                fs.write_bytes(path=self.connection_config.table_uri, value=b"")
+                s3_client.put_object(Bucket=bucket_name, Key=dir_path, Body=b"")
+
+                response = s3_client.get_bucket_location(Bucket=bucket_name)
+
+                if self.connection_config.aws_region != response.get("LocationConstraint"):
+                    raise ValueError("Wrong AWS Region was provided.")
 
             except Exception as e:
                 logger.error(f"failed to validate connection: {e}", exc_info=True)

unstructured_ingest/v2/processes/connectors/elasticsearch.py

@@ -191,6 +191,7 @@ class ElasticsearchIndexer(Indexer):
             yield FileData(
                 identifier=identified,
                 connector_type=CONNECTOR_TYPE,
+                doc_type="batch",
                 metadata=FileDataSourceMetadata(
                     url=f"{self.connection_config.hosts[0]}/{self.index_config.index_name}",
                     date_processed=str(time()),

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
+import os
 import random
+import shutil
+import tempfile
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
@@ -207,12 +210,35 @@ class FsspecDownloader(Downloader):
             **self.connection_config.get_access_config(),
         )
 
+    def handle_directory_download(self, lpath: Path) -> None:
+        # If the object's name contains certain characters (i.e. '?'), it
+        # gets downloaded into a new directory of the same name. This
+        # reconciles that with what is expected, which is to download it
+        # as a file that is not within a directory.
+        if not lpath.is_dir():
+            return
+        desired_name = lpath.name
+        files_in_dir = [file for file in lpath.iterdir() if file.is_file()]
+        if not files_in_dir:
+            raise ValueError(f"no files in {lpath}")
+        if len(files_in_dir) > 1:
+            raise ValueError(
+                "Multiple files in {}: {}".format(lpath, ", ".join([str(f) for f in files_in_dir]))
+            )
+        file = files_in_dir[0]
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_location = os.path.join(temp_dir, desired_name)
+            shutil.copyfile(src=file, dst=temp_location)
+            shutil.rmtree(lpath)
+            shutil.move(src=temp_location, dst=lpath)
+
     def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             self.fs.get(rpath=rpath, lpath=download_path.as_posix())
+            self.handle_directory_download(lpath=download_path)
         except Exception as e:
             logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
             raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
@@ -224,6 +250,7 @@ class FsspecDownloader(Downloader):
         try:
             rpath = file_data.additional_metadata["original_file_path"]
             await self.fs.get(rpath=rpath, lpath=download_path.as_posix())
+            self.handle_directory_download(lpath=download_path)
         except Exception as e:
             logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
             raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
@@ -231,9 +258,7 @@ class FsspecDownloader(Downloader):
 
 
 class FsspecUploaderConfig(FileConfig, UploaderConfig):
-    overwrite: bool = Field(
-        default=False, description="If true, an existing file will be overwritten."
-    )
+    pass
 
 
 FsspecUploaderConfigT = TypeVar("FsspecUploaderConfigT", bound=FsspecUploaderConfig)
@@ -288,9 +313,6 @@ class FsspecUploader(Uploader):
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         path_str = str(path.resolve())
         upload_path = self.get_upload_path(file_data=file_data)
-        if self.fs.exists(path=str(upload_path)) and not self.upload_config.overwrite:
-            logger.debug(f"skipping upload of {path} to {upload_path}, file already exists")
-            return
         logger.debug(f"writing local file {path_str} to {upload_path}")
         self.fs.upload(lpath=path_str, rpath=str(upload_path))
 
@@ -298,9 +320,5 @@ class FsspecUploader(Uploader):
         upload_path = self.get_upload_path(file_data=file_data)
         path_str = str(path.resolve())
         # Odd that fsspec doesn't run exists() as async even when client support async
-        already_exists = self.fs.exists(path=str(upload_path))
-        if already_exists and not self.upload_config.overwrite:
-            logger.debug(f"skipping upload of {path} to {upload_path}, file already exists")
-            return
         logger.debug(f"writing local file {path_str} to {upload_path}")
         self.fs.upload(lpath=path_str, rpath=str(upload_path))

unstructured_ingest/v2/processes/connectors/gitlab.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+from urllib.parse import urlparse
+
+from pydantic import Field, Secret, model_validator
+
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    FileData,
+    FileDataSourceMetadata,
+    Indexer,
+    IndexerConfig,
+    SourceIdentifiers,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+CONNECTOR_TYPE = "gitlab"
+if TYPE_CHECKING:
+    from gitlab import Gitlab
+    from gitlab.v4.objects.projects import Project
+
+
+class GitLabAccessConfig(AccessConfig):
+    access_token: Optional[str] = Field(
+        default=None,
+        description="Optional personal access token for authenticating with the GitLab API.",
+    )
+
+
+class GitLabConnectionConfig(ConnectionConfig):
+    access_config: Secret[GitLabAccessConfig] = Field(
+        default_factory=GitLabAccessConfig,
+        validate_default=True,
+        description="Secret configuration for accessing the GitLab API by authentication token.",
+    )
+    url: str = Field(description="The full URL to the GitLab project or repository.")
+    base_url: str = Field(
+        default="https://gitlab.com",
+        description="The base URL for the GitLab instance (default is GitLab's public domain).",
+    )
+    repo_path: str = Field(
+        default=None,
+        init=False,
+        repr=False,
+        description="The normalized path extracted from the repository URL.",
+    )
+
+    @model_validator(mode="after")
+    def set_repo_path(self):
+        """
+        Parses the provided GitLab URL to extract the `base_url` and `repo_path`,
+        ensuring both are properly formatted for use.
+
+        If the URL contains a scheme (e.g., 'https') and a network location (e.g., 'gitlab.com'),
+        the `base_url` is set accordingly. The repository path is extracted and normalized
+        by removing any leading slashes.
+
+        Notes:
+            - If the URL contains both a scheme and network location, the `base_url` is
+              extracted directly from the URL.
+            - The `repo_path` is adjusted to remove any leading slashes.
+            - This method assumes that the URL follows GitLab's structure
+              (e.g., 'https://gitlab.com/owner/repo').
+        """
+        parsed_gh_url = urlparse(self.url)
+
+        if parsed_gh_url.scheme and parsed_gh_url.netloc:
+            self.base_url = f"{parsed_gh_url.scheme}://{parsed_gh_url.netloc}"
+        self.repo_path = parsed_gh_url.path.lstrip("/")
+
+        return self
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["gitlab"], extras="gitlab")
+    def get_client(self) -> "Gitlab":
+        from gitlab import Gitlab
+
+        logger.info(f"Connection to GitLab: {self.base_url!r}")
+        gitlab = Gitlab(
+            self.base_url, private_token=self.access_config.get_secret_value().access_token
+        )
+        return gitlab
+
+    def get_project(self) -> "Project":
+        """Retrieves the specified GitLab project using the configured base URL and access token.
+
+        Returns:
+            Project: A GitLab `Project` object representing the specified repository.
+
+        Raises:
+            SourceConnectionError: If the GitLab API connection fails.
+            gitlab.exceptions.GitlabGetError: If the project is not found.
+        """
+        gitlab = self.get_client()
+
+        logger.info(f"Accessing Project: '{self.repo_path}'")
+        project = gitlab.projects.get(self.repo_path)
+
+        logger.info(f"Successfully accessed project '{self.repo_path}'")
+        return project
+
+
+class GitLabIndexerConfig(IndexerConfig):
+    path: Path = Field(
+        default="/", description=("Path to the location in the repository that will be processed.")
+    )
+    recursive: bool = Field(
+        default=True,
+        description=(
+            "Flag to control recursive operations when indexing. "
+            "If True, the indexer will traverse directories recursively."
+        ),
+    )
+    git_branch: Optional[str] = Field(
+        default=None,
+        description="The name of the branch to interact with.",
+    )
+
+
+@dataclass
+class GitLabIndexer(Indexer):
+    connection_config: GitLabConnectionConfig
+    index_config: GitLabIndexerConfig
+
+    def precheck(self) -> None:
+        """Validates the connection to the GitLab instance by authenticating or
+        accessing the project.
+
+        This method ensures that the GitLab credentials and configuration are correct by
+        either authenticating or attempting to fetch the specified project.
+
+        Raises:
+            SourceConnectionError: If the connection or authentication with GitLab fails.
+        """
+
+        try:
+            gitlab = self.connection_config.get_client()
+            if self.connection_config.access_config.get_secret_value().access_token is not None:
+                gitlab.auth()
+            else:
+                gitlab.projects.get(self.connection_config.repo_path)
+
+        except Exception as e:
+            logger.error(f"Failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"Failed to validate connection: {e}")
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        """Iterates over the GitLab repository tree and yields file metadata as `FileData` objects.
+
+        This method fetches the repository tree for the specified branch and iterates
+        over its contents. For each file (blob), it generates a `FileData` object containing
+        the file's metadata, path, and permissions.
+
+        Args:
+            **kwargs (Any): Additional keyword arguments (if required).
+
+        Yields:
+            FileData: A generator that yields `FileData` objects representing each file (blob)
+            in the repository.
+        """
+        project = self.connection_config.get_project()
+
+        ref = self.index_config.git_branch or project.default_branch
+
+        files = project.repository_tree(
+            path=str(self.index_config.path),
+            ref=ref,
+            recursive=self.index_config.recursive,
+            iterator=True,
+            all=True,
+        )
+
+        for file in files:
+            relative_path = str(Path(file["path"]).relative_to(self.index_config.path))
+            if file["type"] == "blob":
+                record_locator = {
+                    "file_path": file["path"],
+                    "ref": ref,
+                }
+
+                yield FileData(
+                    identifier=file["id"],
+                    connector_type=CONNECTOR_TYPE,
+                    source_identifiers=SourceIdentifiers(
+                        fullpath=file["path"],
+                        filename=Path(file["path"]).name,
+                        rel_path=relative_path,
+                    ),
+                    metadata=FileDataSourceMetadata(
+                        url=file["id"],
+                        record_locator=record_locator,
+                        permissions_data=[{"mode": file["mode"]}],
+                    ),
+                    additional_metadata={},
+                )
+
+
+class GitLabDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class GitLabDownloader(Downloader):
+    connection_config: GitLabConnectionConfig
+    download_config: GitLabDownloaderConfig
+
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        """Downloads a file from the repository and returns a `DownloadResponse`.
+
+        Args:
+            file_data (FileData): Metadata about the file to be downloaded.
+            **kwargs (Any): Additional arguments (if required).
+
+        Returns:
+            DownloadResponse: A response object containing the download details.
+        """
+        download_path = self.get_download_path(file_data=file_data)
+        if download_path is None:
+            logger.error(
+                "Generated download path is None, source_identifiers might be missing"
+                "from FileData."
+            )
+            raise ValueError("Generated invalid download path.")
+
+        self._download_file(file_data, download_path)
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+    def _download_file(self, file_data: FileData, download_path: Path) -> None:
+        # NOTE: Indexer should supply the record locator in metadata
+        if (
+            file_data.metadata.record_locator is None
+            or "ref" not in file_data.metadata.record_locator
+            or "file_path" not in file_data.metadata.record_locator
+        ):
+            logger.error(
+                f"Invalid record locator in metadata: {file_data.metadata.record_locator}."
+                "Keys 'ref' and 'path' must be present."
+            )
+            raise ValueError("Invalid record locator.")
+
+        ref = file_data.metadata.record_locator["ref"]
+        path = file_data.metadata.record_locator["file_path"]
+
+        project_file = self.connection_config.get_project().files.get(file_path=path, ref=ref)
+        download_path.parent.mkdir(exist_ok=True, parents=True)
+
+        with open(download_path, "wb") as file:
+            file.write(project_file.decode())
+
+
+gitlab_source_entry = SourceRegistryEntry(
+    connection_config=GitLabConnectionConfig,
+    indexer_config=GitLabIndexerConfig,
+    indexer=GitLabIndexer,
+    downloader_config=GitLabDownloaderConfig,
+    downloader=GitLabDownloader,
+)

unstructured_ingest/v2/processes/connectors/google_drive.py

@@ -19,12 +19,12 @@ from unstructured_ingest.v2.interfaces import (
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
+    DownloadResponse,
     FileData,
     FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
-    download_responses,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
@@ -294,7 +294,7 @@ class GoogleDriveDownloader(Downloader):
             _, downloaded = downloader.next_chunk()
         return downloaded
 
-    def _write_file(self, file_data: FileData, file_contents: io.BytesIO):
+    def _write_file(self, file_data: FileData, file_contents: io.BytesIO) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         logger.debug(f"writing {file_data.source_identifiers.fullpath} to {download_path}")
@@ -303,7 +303,7 @@ class GoogleDriveDownloader(Downloader):
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
     @requires_dependencies(["googleapiclient"], extras="google-drive")
-    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         from googleapiclient.http import MediaIoBaseDownload
 
         logger.debug(f"fetching file: {file_data.source_identifiers.fullpath}")

unstructured_ingest/v2/processes/connectors/kafka/__init__.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .cloud import CONNECTOR_TYPE as CLOUD_CONNECTOR
+from .cloud import kafka_cloud_destination_entry, kafka_cloud_source_entry
+from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR
+from .local import kafka_local_destination_entry, kafka_local_source_entry
+
+add_source_entry(source_type=LOCAL_CONNECTOR, entry=kafka_local_source_entry)
+add_destination_entry(destination_type=LOCAL_CONNECTOR, entry=kafka_local_destination_entry)
+
+add_source_entry(source_type=CLOUD_CONNECTOR, entry=kafka_cloud_source_entry)
+add_destination_entry(destination_type=CLOUD_CONNECTOR, entry=kafka_cloud_destination_entry)

unstructured_ingest/v2/processes/connectors/kafka/cloud.py

@@ -0,0 +1,118 @@
+import socket
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Optional
+
+from pydantic import Field, Secret, SecretStr
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.kafka.kafka import (
+    KafkaAccessConfig,
+    KafkaConnectionConfig,
+    KafkaDownloader,
+    KafkaDownloaderConfig,
+    KafkaIndexer,
+    KafkaIndexerConfig,
+    KafkaUploader,
+    KafkaUploaderConfig,
+)
+
+if TYPE_CHECKING:
+    pass
+
+CONNECTOR_TYPE = "kafka-cloud"
+
+
+class CloudKafkaAccessConfig(KafkaAccessConfig):
+    api_key: Optional[SecretStr] = Field(
+        description="Kafka API key to connect at the server", alias="kafka_api_key", default=None
+    )
+    secret: Optional[SecretStr] = Field(description="", default=None)
+
+
+class CloudKafkaConnectionConfig(KafkaConnectionConfig):
+    access_config: Secret[CloudKafkaAccessConfig]
+
+    def get_consumer_configuration(self) -> dict:
+        bootstrap = self.bootstrap_server
+        port = self.port
+        access_config = self.access_config.get_secret_value()
+
+        conf = {
+            "bootstrap.servers": f"{bootstrap}:{port}",
+            "client.id": socket.gethostname(),
+            "group.id": "default_group_id",
+            "enable.auto.commit": "false",
+            "auto.offset.reset": "earliest",
+            "sasl.username": access_config.api_key,
+            "sasl.password": access_config.secret,
+            "sasl.mechanism": "PLAIN",
+            "security.protocol": "SASL_SSL",
+        }
+
+        return conf
+
+    def get_producer_configuration(self) -> dict:
+        bootstrap = self.bootstrap_server
+        port = self.port
+        access_config = self.access_config.get_secret_value()
+
+        conf = {
+            "bootstrap.servers": f"{bootstrap}:{port}",
+            "sasl.username": access_config.api_key,
+            "sasl.password": access_config.secret,
+            "sasl.mechanism": "PLAIN",
+            "security.protocol": "SASL_SSL",
+        }
+
+        return conf
+
+
+class CloudKafkaIndexerConfig(KafkaIndexerConfig):
+    pass
+
+
+@dataclass
+class CloudKafkaIndexer(KafkaIndexer):
+    connection_config: CloudKafkaConnectionConfig
+    index_config: CloudKafkaIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class CloudKafkaDownloaderConfig(KafkaDownloaderConfig):
+    pass
+
+
+@dataclass
+class CloudKafkaDownloader(KafkaDownloader):
+    connection_config: CloudKafkaConnectionConfig
+    download_config: CloudKafkaDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class CloudKafkaUploaderConfig(KafkaUploaderConfig):
+    pass
+
+
+@dataclass
+class CloudKafkaUploader(KafkaUploader):
+    connection_config: CloudKafkaConnectionConfig
+    upload_config: CloudKafkaUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+kafka_cloud_source_entry = SourceRegistryEntry(
+    connection_config=CloudKafkaConnectionConfig,
+    indexer=CloudKafkaIndexer,
+    indexer_config=CloudKafkaIndexerConfig,
+    downloader=CloudKafkaDownloader,
+    downloader_config=CloudKafkaDownloaderConfig,
+)
+
+kafka_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=CloudKafkaConnectionConfig,
+    uploader=CloudKafkaUploader,
+    uploader_config=CloudKafkaUploaderConfig,
+)