unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/connector/fsspec/dropbox.py

@@ -0,0 +1,160 @@
+"""
+Dropbox Connector
+The Dropbox Connector presents a couple abnormal situations.
+1) They don't have an unexpiring token
+2) They require a forward slash `/` in front of the remote_file_path. This presents
+some real problems creating paths. When appending a path that begins with a
+forward slash to any path, whether using the / shorthand or joinpath, causes the
+starting path to disappear. So the `/` needs to be stripped off.
+3) To list and get files from the root directory Dropbox you need a ""," ", or " /"
+"""
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Type
+
+from unstructured_ingest.connector.fsspec.fsspec import (
+    FsspecDestinationConnector,
+    FsspecIngestDoc,
+    FsspecSourceConnector,
+    FsspecWriteConfig,
+    SimpleFsspecConfig,
+)
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.interfaces import AccessConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+
+class MissingFolderError(Exception):
+    """There is no folder by that name. For root try `dropbox:// /`"""
+
+
+@dataclass
+class DropboxAccessConfig(AccessConfig):
+    token: str = enhanced_field(sensitive=True)
+
+
+@dataclass
+class DropboxWriteConfig(FsspecWriteConfig):
+    pass
+
+
+@dataclass
+class SimpleDropboxConfig(SimpleFsspecConfig):
+    access_config: DropboxAccessConfig = None
+
+
+@dataclass
+class DropboxIngestDoc(FsspecIngestDoc):
+    connector_config: SimpleDropboxConfig
+    registry_name: str = "dropbox"
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    def get_file(self):
+        super().get_file()
+
+    @property
+    def _output_filename(self):
+        # Dropbox requires a forward slash at the front of the folder path. This
+        # creates some complications in path joining so a custom path is created here.
+        # Dropbox uses an empty string `""`, or a space `" "`` or a `" /"` to list root
+        if self.connector_config.dir_path == " ":
+            return Path(self.processor_config.output_dir) / re.sub(
+                "^/",
+                "",
+                f"{self.remote_file_path}.json",
+            )
+        else:
+            return (
+                Path(self.processor_config.output_dir)
+                / f"{self.remote_file_path.replace(f'/{self.connector_config.dir_path}/', '')}.json"
+            )
+
+    def _tmp_download_file(self):
+        # Dropbox requires a forward slash at the front of the folder path. This
+        # creates some complications in path joining so a custom path is created here.
+        # Dropbox uses an empty string `""`, or a space `" "`` or a `" /"` to list root
+        download_dir: str = self.read_config.download_dir if self.read_config.download_dir else ""
+        if not download_dir:
+            return ""
+        if self.connector_config.dir_path == " ":
+            return Path(download_dir) / re.sub(
+                "^/",
+                "",
+                self.remote_file_path,
+            )
+        else:
+            return Path(download_dir) / self.remote_file_path.replace(
+                f"/{self.connector_config.dir_path}/",
+                "",
+            )
+
+
+@dataclass
+class DropboxSourceConnector(FsspecSourceConnector):
+    connector_config: SimpleDropboxConfig
+
+    def __post_init__(self):
+        self.ingest_doc_cls: Type[DropboxIngestDoc] = DropboxIngestDoc
+
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    def initialize(self):
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        try:
+            self.fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+                **self.connector_config.get_access_config(),
+            )
+            # Dropbox requires a forward slash at the front of the folder path. This
+            # creates some complications in path joining so a custom path is created here.
+            ls_output = self.fs.ls(f"/{self.connector_config.path_without_protocol}")
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+        if ls_output and len(ls_output) >= 1:
+            return
+        elif ls_output:
+            raise ValueError(
+                f"No objects found in {self.connector_config.remote_url}.",
+            )
+        else:
+            raise MissingFolderError(
+                "There is no folder by that name. For root try `dropbox:// /`",
+            )
+
+    def _list_files(self):
+        # Dropbox requires a forward slash at the front of the folder path. This
+        # creates some complications in path joining so a custom path is created here.
+        if not self.connector_config.recursive:
+            # fs.ls does not walk directories
+            # directories that are listed in cloud storage can cause problems because they are seen
+            # as 0byte files
+            return [
+                x.get("name")
+                for x in self.fs.ls(
+                    f"/{self.connector_config.path_without_protocol}",
+                    detail=True,
+                )
+                if x.get("size")
+            ]
+        else:
+            # fs.find will recursively walk directories
+            # "size" is a common key for all the cloud protocols with fs
+            return [
+                k
+                for k, v in self.fs.find(
+                    f"/{self.connector_config.path_without_protocol}",
+                    detail=True,
+                ).items()
+                if v.get("size")
+            ]
+
+
+@dataclass
+class DropboxDestinationConnector(FsspecDestinationConnector):
+    connector_config: SimpleFsspecConfig
+    write_config: DropboxWriteConfig

unstructured_ingest/connector/fsspec/fsspec.py

@@ -0,0 +1,359 @@
+import fnmatch
+import json
+import os
+import typing as t
+from abc import ABC
+from contextlib import suppress
+from dataclasses import dataclass
+from pathlib import Path, PurePath
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+)
+from unstructured_ingest.interfaces import (
+    BaseConnectorConfig,
+    BaseDestinationConnector,
+    BaseSingleIngestDoc,
+    BaseSourceConnector,
+    FsspecConfig,
+    IngestDocCleanupMixin,
+    SourceConnectorCleanupMixin,
+    SourceMetadata,
+    WriteConfig,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.compression import (
+    TAR_FILE_EXT,
+    ZIP_FILE_EXT,
+    CompressionSourceConnectorMixin,
+)
+from unstructured_ingest.utils.dep_check import (
+    requires_dependencies,
+)
+
+SUPPORTED_REMOTE_FSSPEC_PROTOCOLS = [
+    "s3",
+    "s3a",
+    "abfs",
+    "az",
+    "gs",
+    "gcs",
+    "box",
+    "dropbox",
+    "sftp",
+]
+
+
+@dataclass
+class SimpleFsspecConfig(FsspecConfig, BaseConnectorConfig):
+    pass
+
+
+@dataclass
+class FsspecIngestDoc(IngestDocCleanupMixin, BaseSingleIngestDoc):
+    """Class encapsulating fetching a doc and writing processed results (but not
+    doing the processing!).
+
+    Also includes a cleanup method. When things go wrong and the cleanup
+    method is not called, the file is left behind on the filesystem to assist debugging.
+    """
+
+    connector_config: SimpleFsspecConfig
+    remote_file_path: str
+
+    def _tmp_download_file(self):
+        download_dir = self.read_config.download_dir if self.read_config.download_dir else ""
+        return Path(download_dir) / self.remote_file_path.replace(
+            f"{self.connector_config.dir_path}/",
+            "",
+        )
+
+    @property
+    def _output_filename(self):
+        # Dynamically parse filename , can change if remote path was pointing to the single
+        # file, a directory, or nested directory
+        if self.remote_file_path == self.connector_config.path_without_protocol:
+            file = self.remote_file_path.split("/")[-1]
+            filename = f"{file}.json"
+        else:
+            path_without_protocol = (
+                self.connector_config.path_without_protocol
+                if self.connector_config.path_without_protocol.endswith("/")
+                else f"{self.connector_config.path_without_protocol}/"
+            )
+            filename = f"{self.remote_file_path.replace(path_without_protocol, '')}.json"
+        return Path(self.processor_config.output_dir) / filename
+
+    def _create_full_tmp_dir_path(self):
+        """Includes "directories" in the object path"""
+        self._tmp_download_file().parent.mkdir(parents=True, exist_ok=True)
+
+    @SourceConnectionError.wrap
+    @BaseSingleIngestDoc.skip_if_file_exists
+    def get_file(self):
+        """Fetches the file from the current filesystem and stores it locally."""
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        self._create_full_tmp_dir_path()
+        fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+            **self.connector_config.get_access_config(),
+        )
+        self._get_file(fs=fs)
+        fs.get(rpath=self.remote_file_path, lpath=self._tmp_download_file().as_posix())
+        self.update_source_metadata()
+
+    @SourceConnectionNetworkError.wrap
+    def _get_file(self, fs):
+        fs.get(rpath=self.remote_file_path, lpath=self._tmp_download_file().as_posix())
+
+    @requires_dependencies(["fsspec"])
+    def update_source_metadata(self):
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+            **self.connector_config.get_access_config(),
+        )
+
+        date_created = None
+        with suppress(NotImplementedError):
+            date_created = fs.created(self.remote_file_path).isoformat()
+
+        date_modified = None
+        with suppress(NotImplementedError):
+            date_modified = fs.modified(self.remote_file_path).isoformat()
+
+        version = (
+            fs.checksum(self.remote_file_path)
+            if self.connector_config.protocol != "gs"
+            else fs.info(self.remote_file_path).get("etag", "")
+        )
+        file_exists = fs.exists(self.remote_file_path)
+        self.source_metadata = SourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            version=str(version),
+            source_url=f"{self.connector_config.protocol}://{self.remote_file_path}",
+            exists=file_exists,
+        )
+
+    @property
+    def filename(self):
+        """The filename of the file after downloading from cloud"""
+        return self._tmp_download_file()
+
+    @property
+    def record_locator(self) -> t.Optional[t.Dict[str, t.Any]]:
+        """Returns the equivalent of ls in dict"""
+        return {
+            "protocol": self.connector_config.protocol,
+            "remote_file_path": self.remote_file_path,
+        }
+
+
+@dataclass
+class FsspecSourceConnector(
+    SourceConnectorCleanupMixin,
+    CompressionSourceConnectorMixin,
+    BaseSourceConnector,
+):
+    """Objects of this class support fetching document(s) from"""
+
+    connector_config: SimpleFsspecConfig
+
+    def check_connection(self):
+        from fsspec import get_filesystem_class
+
+        try:
+            fs = get_filesystem_class(self.connector_config.protocol)(
+                **self.connector_config.get_access_config(),
+            )
+            fs.ls(path=self.connector_config.path_without_protocol, detail=False)
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def __post_init__(self):
+        self.ingest_doc_cls: t.Type[FsspecIngestDoc] = FsspecIngestDoc
+
+    def initialize(self):
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        self.fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+            **self.connector_config.get_access_config(),
+        )
+
+        """Verify that can get metadata for an object, validates connections info."""
+        ls_output = self.fs.ls(self.connector_config.path_without_protocol, detail=False)
+        if len(ls_output) < 1:
+            raise ValueError(
+                f"No objects found in {self.connector_config.remote_url}.",
+            )
+
+    def _list_files(self):
+        if not self.connector_config.recursive:
+            # fs.ls does not walk directories
+            # directories that are listed in cloud storage can cause problems
+            # because they are seen as 0 byte files
+            return [
+                x.get("name")
+                for x in self.fs.ls(self.connector_config.path_without_protocol, detail=True)
+                if x.get("size") > 0
+            ]
+        else:
+            # fs.find will recursively walk directories
+            # "size" is a common key for all the cloud protocols with fs
+            return [
+                k
+                for k, v in self.fs.find(
+                    self.connector_config.path_without_protocol,
+                    detail=True,
+                ).items()
+                if v.get("size") > 0
+            ]
+
+    def does_path_match_glob(self, path: str) -> bool:
+        if self.connector_config.file_glob is None:
+            return True
+        patterns = self.connector_config.file_glob
+        for pattern in patterns:
+            if fnmatch.filter([path], pattern):
+                return True
+        logger.debug(f"The file {path!r} is discarded as it does not match any given glob.")
+        return False
+
+    def get_ingest_docs(self):
+        raw_files = self._list_files()
+        # If glob filters provided, use to fiter on filepaths
+        files = [f for f in raw_files if self.does_path_match_glob(f)]
+        # remove compressed files
+        compressed_file_ext = TAR_FILE_EXT + ZIP_FILE_EXT
+        compressed_files = []
+        uncompressed_files = []
+        docs: t.List[BaseSingleIngestDoc] = []
+        for file in files:
+            if any(file.endswith(ext) for ext in compressed_file_ext):
+                compressed_files.append(file)
+            else:
+                uncompressed_files.append(file)
+        docs.extend(
+            [
+                self.ingest_doc_cls(
+                    read_config=self.read_config,
+                    connector_config=self.connector_config,
+                    processor_config=self.processor_config,
+                    remote_file_path=file,
+                )
+                for file in uncompressed_files
+            ],
+        )
+        if not self.connector_config.uncompress:
+            return docs
+        for compressed_file in compressed_files:
+            compressed_doc = self.ingest_doc_cls(
+                read_config=self.read_config,
+                processor_config=self.processor_config,
+                connector_config=self.connector_config,
+                remote_file_path=compressed_file,
+            )
+            try:
+                local_ingest_docs = self.process_compressed_doc(doc=compressed_doc)
+                logger.info(f"adding {len(local_ingest_docs)} from {compressed_file}")
+                docs.extend(local_ingest_docs)
+            finally:
+                compressed_doc.cleanup_file()
+        return docs
+
+
+@dataclass
+class WriteTextConfig(EnhancedDataClassJsonMixin, ABC):
+    pass
+
+
+@dataclass
+class FsspecWriteConfig(WriteConfig):
+    write_text_config: t.Optional[WriteTextConfig] = None
+
+    def get_write_text_config(self) -> t.Dict[str, t.Any]:
+        if write_text_kwargs := self.write_text_config:
+            return write_text_kwargs.to_dict()
+        return {}
+
+
+@dataclass
+class FsspecDestinationConnector(BaseDestinationConnector):
+    connector_config: SimpleFsspecConfig
+    write_config: FsspecWriteConfig
+
+    def initialize(self):
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        self.fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+            **self.connector_config.get_access_config(),
+        )
+        self.check_connection()
+
+    def check_connection(self):
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        try:
+            fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+                **self.connector_config.get_access_config(),
+            )
+
+            # e.g. Dropbox path starts with /
+            bucket_name = "/" if self.connector_config.path_without_protocol.startswith("/") else ""
+            bucket_name += self.connector_config.dir_path.split("/")[0]
+
+            logger.info(f"checking connection for destination {bucket_name}")
+            fs.ls(path=bucket_name, detail=False)
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    def write_dict(
+        self,
+        *args,
+        elements_dict: t.List[t.Dict[str, t.Any]],
+        filename: t.Optional[str] = None,
+        indent: int = 4,
+        encoding: str = "utf-8",
+        **kwargs,
+    ) -> None:
+        from fsspec import AbstractFileSystem, get_filesystem_class
+
+        fs: AbstractFileSystem = get_filesystem_class(self.connector_config.protocol)(
+            **self.connector_config.get_access_config(),
+        )
+
+        logger.info(f"Writing content using filesystem: {type(fs).__name__}")
+
+        output_folder = self.connector_config.path_without_protocol
+        output_folder = os.path.join(output_folder)  # Make sure folder ends with file seperator
+        filename = (
+            filename.strip(os.sep) if filename else filename
+        )  # Make sure filename doesn't begin with file seperator
+        output_path = str(PurePath(output_folder, filename)) if filename else output_folder
+        full_output_path = f"{self.connector_config.protocol}://{output_path}"
+        logger.debug(f"uploading content to {full_output_path}")
+        write_text_configs = self.write_config.get_write_text_config() if self.write_config else {}
+        fs.write_text(
+            full_output_path,
+            json.dumps(elements_dict, indent=indent),
+            encoding=encoding,
+            **write_text_configs,
+        )
+
+    def get_elements_dict(self, docs: t.List[BaseSingleIngestDoc]) -> t.List[t.Dict[str, t.Any]]:
+        pass
+
+    def write(self, docs: t.List[BaseSingleIngestDoc]) -> None:
+        for doc in docs:
+            file_path = doc.base_output_filename
+            filename = file_path if file_path else None
+            with open(doc._output_filename) as json_file:
+                logger.debug(f"uploading content from {doc._output_filename}")
+                json_list = json.load(json_file)
+                self.write_dict(elements_dict=json_list, filename=filename)

unstructured_ingest/connector/fsspec/gcs.py

@@ -0,0 +1,82 @@
+import typing as t
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Type
+
+from unstructured_ingest.connector.fsspec.fsspec import (
+    FsspecDestinationConnector,
+    FsspecIngestDoc,
+    FsspecSourceConnector,
+    FsspecWriteConfig,
+    SimpleFsspecConfig,
+)
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.interfaces import AccessConfig
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.string_and_date_utils import json_to_dict
+
+
+@dataclass
+class GcsAccessConfig(AccessConfig):
+    token: t.Optional[str] = enhanced_field(
+        default=None, sensitive=True, overload_name="service_account_key"
+    )
+
+    def __post_init__(self):
+        ALLOWED_AUTH_VALUES = "google_default", "cache", "anon", "browser", "cloud"
+
+        # Case: null value
+        if not self.token:
+            return
+        # Case: one of auth constants
+        if self.token in ALLOWED_AUTH_VALUES:
+            return
+        # Case: token as json
+        if isinstance(json_to_dict(self.token), dict):
+            self.token = json_to_dict(self.token)
+            return
+        # Case: path to token
+        if Path(self.token).is_file():
+            return
+
+        raise ValueError("Invalid auth token value")
+
+
+@dataclass
+class GcsWriteConfig(FsspecWriteConfig):
+    pass
+
+
+@dataclass
+class SimpleGcsConfig(SimpleFsspecConfig):
+    access_config: GcsAccessConfig = None
+
+
+@dataclass
+class GcsIngestDoc(FsspecIngestDoc):
+    connector_config: SimpleGcsConfig
+    registry_name: str = "gcs"
+
+    @SourceConnectionError.wrap
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def get_file(self):
+        super().get_file()
+
+
+@dataclass
+class GcsSourceConnector(FsspecSourceConnector):
+    connector_config: SimpleGcsConfig
+
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def initialize(self):
+        super().initialize()
+
+    def __post_init__(self):
+        self.ingest_doc_cls: Type[GcsIngestDoc] = GcsIngestDoc
+
+
+@dataclass
+class GcsDestinationConnector(FsspecDestinationConnector):
+    connector_config: SimpleGcsConfig
+    write_config: GcsWriteConfig

unstructured_ingest/connector/fsspec/s3.py

@@ -0,0 +1,62 @@
+import typing as t
+from dataclasses import dataclass
+from typing import Type
+
+from unstructured_ingest.connector.fsspec.fsspec import (
+    FsspecDestinationConnector,
+    FsspecIngestDoc,
+    FsspecSourceConnector,
+    FsspecWriteConfig,
+    SimpleFsspecConfig,
+)
+from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.interfaces import AccessConfig
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+
+@dataclass
+class S3AccessConfig(AccessConfig):
+    anon: bool = enhanced_field(default=False, overload_name="anonymous")
+    endpoint_url: t.Optional[str] = None
+    key: t.Optional[str] = enhanced_field(default=None, sensitive=True)
+    secret: t.Optional[str] = enhanced_field(default=None, sensitive=True)
+    token: t.Optional[str] = enhanced_field(default=None, sensitive=True)
+
+
+@dataclass
+class S3WriteConfig(FsspecWriteConfig):
+    pass
+
+
+@dataclass
+class SimpleS3Config(SimpleFsspecConfig):
+    access_config: S3AccessConfig = enhanced_field(default=None)
+
+
+@dataclass
+class S3IngestDoc(FsspecIngestDoc):
+    connector_config: SimpleS3Config
+    remote_file_path: str
+    registry_name: str = "s3"
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def get_file(self):
+        super().get_file()
+
+
+@dataclass
+class S3SourceConnector(FsspecSourceConnector):
+    connector_config: SimpleS3Config
+
+    def __post_init__(self):
+        self.ingest_doc_cls: Type[S3IngestDoc] = S3IngestDoc
+
+
+@dataclass
+class S3DestinationConnector(FsspecDestinationConnector):
+    connector_config: SimpleS3Config
+    write_config: S3WriteConfig
+
+    @requires_dependencies(["s3fs", "fsspec"], extras="s3")
+    def initialize(self):
+        super().initialize()