unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/runner/mongodb.py

@@ -0,0 +1,34 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.mongodb import SimpleMongoDBConfig
+
+
+@dataclass
+class MongoDBRunner(Runner):
+    connector_config: "SimpleMongoDBConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            str(self.connector_config.access_config.uri).encode("utf-8"),
+        )
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="mongodb",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.mongodb import (
+            MongoDBSourceConnector,
+        )
+
+        return MongoDBSourceConnector

unstructured_ingest/runner/notion.py

@@ -0,0 +1,61 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.notion.connector import SimpleNotionConfig
+
+
+@dataclass
+class NotionRunner(Runner):
+    connector_config: "SimpleNotionConfig"
+
+    def update_read_config(self):
+        if not self.connector_config.page_ids and not self.connector_config.database_ids:
+            raise ValueError("no page ids nor database ids provided")
+
+        if self.connector_config.page_ids and self.connector_config.database_ids:
+            hashed_dir_name = hashlib.sha256(
+                "{},{}".format(
+                    ",".join(self.connector_config.page_ids),
+                    ",".join(self.connector_config.database_ids),
+                ).encode("utf-8"),
+            )
+        elif self.connector_config.page_ids:
+            hashed_dir_name = hashlib.sha256(
+                ",".join(self.connector_config.page_ids).encode("utf-8"),
+            )
+        elif self.connector_config.database_ids:
+            hashed_dir_name = hashlib.sha256(
+                ",".join(self.connector_config.database_ids).encode("utf-8"),
+            )
+        else:
+            raise ValueError("could not create local cache directory name")
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="notion",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.notion.connector import (
+            NotionSourceConnector,
+        )
+
+        return NotionSourceConnector
+
+    def get_source_connector(self) -> BaseSourceConnector:
+        source_connector_cls = self.get_source_connector_cls()
+        return source_connector_cls(
+            processor_config=self.processor_config,
+            connector_config=self.connector_config,
+            read_config=self.read_config,
+            retry_strategy_config=self.retry_strategy_config,
+        )

unstructured_ingest/runner/onedrive.py

@@ -0,0 +1,35 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.onedrive import SimpleOneDriveConfig
+
+
+@dataclass
+class OneDriveRunner(Runner):
+    connector_config: "SimpleOneDriveConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            f"{self.connector_config.tenant}_{self.connector_config.user_pname}".encode("utf-8"),
+        )
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="onedrive",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.onedrive import (
+            OneDriveSourceConnector,
+        )
+
+        return OneDriveSourceConnector

unstructured_ingest/runner/opensearch.py

@@ -0,0 +1,40 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.opensearch import SimpleOpenSearchConfig
+
+
+@dataclass
+class OpenSearchRunner(Runner):
+    connector_config: "SimpleOpenSearchConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            "{}_{}".format(
+                ",".join(self.connector_config.access_config.hosts),
+                self.connector_config.index_name,
+            ).encode(
+                "utf-8",
+            ),
+        )
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="opensearch",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.opensearch import (
+            OpenSearchSourceConnector,
+        )
+
+        return OpenSearchSourceConnector

unstructured_ingest/runner/outlook.py

@@ -0,0 +1,33 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.outlook import SimpleOutlookConfig
+
+
+@dataclass
+class OutlookRunner(Runner):
+    connector_config: "SimpleOutlookConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(self.connector_config.user_email.encode("utf-8"))
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="outlook",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.outlook import (
+            OutlookSourceConnector,
+        )
+
+        return OutlookSourceConnector

unstructured_ingest/runner/reddit.py

@@ -0,0 +1,35 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.reddit import SimpleRedditConfig
+
+
+@dataclass
+class RedditRunner(Runner):
+    connector_config: "SimpleRedditConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            self.connector_config.subreddit_name.encode("utf-8"),
+        )
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="reddit",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.reddit import (
+            RedditSourceConnector,
+        )
+
+        return RedditSourceConnector

unstructured_ingest/runner/salesforce.py

@@ -0,0 +1,33 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.salesforce import SimpleSalesforceConfig
+
+
+@dataclass
+class SalesforceRunner(Runner):
+    connector_config: "SimpleSalesforceConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(self.connector_config.username.encode("utf-8"))
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="salesforce",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.salesforce import (
+            SalesforceSourceConnector,
+        )
+
+        return SalesforceSourceConnector

unstructured_ingest/runner/sharepoint.py

@@ -0,0 +1,35 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.sharepoint import SimpleSharepointConfig
+
+
+@dataclass
+class SharePointRunner(Runner):
+    connector_config: "SimpleSharepointConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            f"{self.connector_config.site}_{self.connector_config.path}".encode("utf-8"),
+        )
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="sharepoint",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.sharepoint import (
+            SharepointSourceConnector,
+        )
+
+        return SharepointSourceConnector

unstructured_ingest/runner/slack.py

@@ -0,0 +1,33 @@
+import hashlib
+import typing as t
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.slack import SimpleSlackConfig
+
+
+class SlackRunner(Runner):
+    connector_config: "SimpleSlackConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            ",".join(self.connector_config.channels).encode("utf-8"),
+        )
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="slack",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.slack import (
+            SlackSourceConnector,
+        )
+
+        return SlackSourceConnector

unstructured_ingest/runner/utils.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from pathlib import Path
+
+from unstructured_ingest.interfaces import (
+    ReadConfig,
+)
+
+
+def update_download_dir_remote_url(
+    connector_name: str,
+    read_config: ReadConfig,
+    remote_url: str,
+    logger: logging.Logger,
+) -> str:
+    hashed_dir_name = hashlib.sha256(remote_url.encode("utf-8"))
+    return update_download_dir_hash(
+        connector_name=connector_name,
+        read_config=read_config,
+        hashed_dir_name=hashed_dir_name,
+        logger=logger,
+    )
+
+
+def update_download_dir_hash(
+    connector_name: str,
+    read_config: ReadConfig,
+    hashed_dir_name: hashlib._Hash,
+    logger: logging.Logger,
+) -> str:
+    if not read_config.download_dir:
+        cache_path = Path.home() / ".cache" / "unstructured" / "ingest"
+        if not cache_path.exists():
+            cache_path.mkdir(parents=True, exist_ok=True)
+        download_dir = cache_path / connector_name / hashed_dir_name.hexdigest()[:10]
+        if read_config.preserve_downloads:
+            logger.warning(
+                f"Preserving downloaded files but download_dir is not specified,"
+                f" using {download_dir}",
+            )
+        new_download_dir = str(download_dir)
+        logger.debug(f"updating download directory to: {new_download_dir}")
+    else:
+        new_download_dir = read_config.download_dir
+    return new_download_dir

unstructured_ingest/runner/wikipedia.py

@@ -0,0 +1,35 @@
+import hashlib
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseSourceConnector
+from unstructured_ingest.logger import logger
+from unstructured_ingest.runner.base_runner import Runner
+from unstructured_ingest.runner.utils import update_download_dir_hash
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.wikipedia import SimpleWikipediaConfig
+
+
+@dataclass
+class WikipediaRunner(Runner):
+    connector_config: "SimpleWikipediaConfig"
+
+    def update_read_config(self):
+        hashed_dir_name = hashlib.sha256(
+            self.connector_config.page_title.encode("utf-8"),
+        )
+
+        self.read_config.download_dir = update_download_dir_hash(
+            connector_name="wikipedia",
+            read_config=self.read_config,
+            hashed_dir_name=hashed_dir_name,
+            logger=logger,
+        )
+
+    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
+        from unstructured_ingest.connector.wikipedia import (
+            WikipediaSourceConnector,
+        )
+
+        return WikipediaSourceConnector

unstructured_ingest/runner/writers/__init__.py

@@ -0,0 +1,48 @@
+import typing as t
+
+from .astradb import AstraDBWriter
+from .azure_ai_search import AzureAiSearchWriter
+from .base_writer import Writer
+from .chroma import ChromaWriter
+from .clarifai import ClarifaiWriter
+from .databricks_volumes import DatabricksVolumesWriter
+from .delta_table import DeltaTableWriter
+from .elasticsearch import ElasticsearchWriter
+from .fsspec.azure import AzureWriter
+from .fsspec.box import BoxWriter
+from .fsspec.dropbox import DropboxWriter
+from .fsspec.gcs import GcsWriter
+from .fsspec.s3 import S3Writer
+from .kafka import KafkaWriter
+from .mongodb import MongodbWriter
+from .opensearch import OpenSearchWriter
+from .pinecone import PineconeWriter
+from .qdrant import QdrantWriter
+from .sql import SqlWriter
+from .vectara import VectaraWriter
+from .weaviate import WeaviateWriter
+
+writer_map: t.Dict[str, t.Type[Writer]] = {
+    "astradb": AstraDBWriter,
+    "azure": AzureWriter,
+    "azure_ai_search": AzureAiSearchWriter,
+    "box": BoxWriter,
+    "chroma": ChromaWriter,
+    "clarifai": ClarifaiWriter,
+    "databricks_volumes": DatabricksVolumesWriter,
+    "delta_table": DeltaTableWriter,
+    "dropbox": DropboxWriter,
+    "elasticsearch": ElasticsearchWriter,
+    "gcs": GcsWriter,
+    "kafka": KafkaWriter,
+    "mongodb": MongodbWriter,
+    "opensearch": OpenSearchWriter,
+    "pinecone": PineconeWriter,
+    "qdrant": QdrantWriter,
+    "s3": S3Writer,
+    "sql": SqlWriter,
+    "vectara": VectaraWriter,
+    "weaviate": WeaviateWriter,
+}
+
+__all__ = ["writer_map"]

unstructured_ingest/runner/writers/astradb.py

@@ -0,0 +1,22 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.astradb import AstraDBWriteConfig, SimpleAstraDBConfig
+
+
+@dataclass
+class AstraDBWriter(Writer, EnhancedDataClassJsonMixin):
+    write_config: "AstraDBWriteConfig"
+    connector_config: "SimpleAstraDBConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.astradb import (
+            AstraDBDestinationConnector,
+        )
+
+        return AstraDBDestinationConnector

unstructured_ingest/runner/writers/azure_ai_search.py

@@ -0,0 +1,24 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.azure_ai_search import (
+        AzureAISearchWriteConfig,
+        SimpleAzureAISearchStorageConfig,
+    )
+
+
+@dataclass
+class AzureAiSearchWriter(Writer):
+    connector_config: "SimpleAzureAISearchStorageConfig"
+    write_config: "AzureAISearchWriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.azure_ai_search import (
+            AzureAISearchDestinationConnector,
+        )
+
+        return AzureAISearchDestinationConnector

unstructured_ingest/runner/writers/base_writer.py

@@ -0,0 +1,26 @@
+import typing as t
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import (
+    BaseConnectorConfig,
+    BaseDestinationConnector,
+    WriteConfig,
+)
+
+
+@dataclass
+class Writer(ABC):
+    connector_config: BaseConnectorConfig
+    write_config: WriteConfig
+
+    @abstractmethod
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        pass
+
+    def get_connector(self, **kwargs) -> BaseDestinationConnector:
+        connector_cls = self.get_connector_cls()
+        return connector_cls(
+            write_config=self.write_config,
+            connector_config=self.connector_config,
+        )

unstructured_ingest/runner/writers/chroma.py

@@ -0,0 +1,22 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.chroma import ChromaWriteConfig, SimpleChromaConfig
+
+
+@dataclass
+class ChromaWriter(Writer, EnhancedDataClassJsonMixin):
+    write_config: "ChromaWriteConfig"
+    connector_config: "SimpleChromaConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.chroma import (
+            ChromaDestinationConnector,
+        )
+
+        return ChromaDestinationConnector

unstructured_ingest/runner/writers/clarifai.py

@@ -0,0 +1,19 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.clarifai import ClarifaiWriteConfig, SimpleClarifaiConfig
+
+
+@dataclass
+class ClarifaiWriter(Writer):
+    write_config: "ClarifaiWriteConfig"
+    connector_config: "SimpleClarifaiConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.clarifai import ClarifaiDestinationConnector
+
+        return ClarifaiDestinationConnector

unstructured_ingest/runner/writers/databricks_volumes.py

@@ -0,0 +1,25 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.databricks_volumes import (
+        DatabricksVolumesWriteConfig,
+        SimpleDatabricksVolumesConfig,
+    )
+
+
+@dataclass
+class DatabricksVolumesWriter(Writer, EnhancedDataClassJsonMixin):
+    write_config: "DatabricksVolumesWriteConfig"
+    connector_config: "SimpleDatabricksVolumesConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.databricks_volumes import (
+            DatabricksVolumesDestinationConnector,
+        )
+
+        return DatabricksVolumesDestinationConnector

unstructured_ingest/runner/writers/delta_table.py

@@ -0,0 +1,24 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.delta_table import (
+        DeltaTableWriteConfig,
+        SimpleDeltaTableConfig,
+    )
+
+
+@dataclass
+class DeltaTableWriter(Writer):
+    write_config: "DeltaTableWriteConfig"
+    connector_config: "SimpleDeltaTableConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.delta_table import (
+            DeltaTableDestinationConnector,
+        )
+
+        return DeltaTableDestinationConnector

unstructured_ingest/runner/writers/elasticsearch.py

@@ -0,0 +1,24 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.elasticsearch import (
+        ElasticsearchWriteConfig,
+        SimpleElasticsearchConfig,
+    )
+
+
+@dataclass
+class ElasticsearchWriter(Writer):
+    connector_config: "SimpleElasticsearchConfig"
+    write_config: "ElasticsearchWriteConfig"
+
+    def get_connector_cls(self) -> BaseDestinationConnector:
+        from unstructured_ingest.connector.elasticsearch import (
+            ElasticsearchDestinationConnector,
+        )
+
+        return ElasticsearchDestinationConnector