unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/runner/kafka.py

@@ -1,34 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.kafka import SimpleKafkaConfig
-
-
-@dataclass
-class KafkaRunner(Runner):
-    connector_config: "SimpleKafkaConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            str(self.connector_config.bootstrap_server).encode("utf-8"),
-        )
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="kafka",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.kafka import (
-            KafkaSourceConnector,
-        )
-
-        return KafkaSourceConnector

unstructured_ingest/runner/local.py

@@ -1,23 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.runner.base_runner import Runner
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.local import SimpleLocalConfig
-
-
-@dataclass
-class LocalRunner(Runner):
-    connector_config: "SimpleLocalConfig"
-
-    def update_read_config(self):
-        pass
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.local import (
-            LocalSourceConnector,
-        )
-
-        return LocalSourceConnector

unstructured_ingest/runner/mongodb.py

@@ -1,34 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.mongodb import SimpleMongoDBConfig
-
-
-@dataclass
-class MongoDBRunner(Runner):
-    connector_config: "SimpleMongoDBConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            str(self.connector_config.access_config.uri).encode("utf-8"),
-        )
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="mongodb",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.mongodb import (
-            MongoDBSourceConnector,
-        )
-
-        return MongoDBSourceConnector

unstructured_ingest/runner/notion.py

@@ -1,61 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.notion.connector import SimpleNotionConfig
-
-
-@dataclass
-class NotionRunner(Runner):
-    connector_config: "SimpleNotionConfig"
-
-    def update_read_config(self):
-        if not self.connector_config.page_ids and not self.connector_config.database_ids:
-            raise ValueError("no page ids nor database ids provided")
-
-        if self.connector_config.page_ids and self.connector_config.database_ids:
-            hashed_dir_name = hashlib.sha256(
-                "{},{}".format(
-                    ",".join(self.connector_config.page_ids),
-                    ",".join(self.connector_config.database_ids),
-                ).encode("utf-8"),
-            )
-        elif self.connector_config.page_ids:
-            hashed_dir_name = hashlib.sha256(
-                ",".join(self.connector_config.page_ids).encode("utf-8"),
-            )
-        elif self.connector_config.database_ids:
-            hashed_dir_name = hashlib.sha256(
-                ",".join(self.connector_config.database_ids).encode("utf-8"),
-            )
-        else:
-            raise ValueError("could not create local cache directory name")
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="notion",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.notion.connector import (
-            NotionSourceConnector,
-        )
-
-        return NotionSourceConnector
-
-    def get_source_connector(self) -> BaseSourceConnector:
-        source_connector_cls = self.get_source_connector_cls()
-        return source_connector_cls(
-            processor_config=self.processor_config,
-            connector_config=self.connector_config,
-            read_config=self.read_config,
-            retry_strategy_config=self.retry_strategy_config,
-        )

unstructured_ingest/runner/onedrive.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.onedrive import SimpleOneDriveConfig
-
-
-@dataclass
-class OneDriveRunner(Runner):
-    connector_config: "SimpleOneDriveConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            f"{self.connector_config.tenant}_{self.connector_config.user_pname}".encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="onedrive",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.onedrive import (
-            OneDriveSourceConnector,
-        )
-
-        return OneDriveSourceConnector

unstructured_ingest/runner/opensearch.py

@@ -1,40 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.opensearch import SimpleOpenSearchConfig
-
-
-@dataclass
-class OpenSearchRunner(Runner):
-    connector_config: "SimpleOpenSearchConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            "{}_{}".format(
-                ",".join(self.connector_config.access_config.hosts),
-                self.connector_config.index_name,
-            ).encode(
-                "utf-8",
-            ),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="opensearch",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.opensearch import (
-            OpenSearchSourceConnector,
-        )
-
-        return OpenSearchSourceConnector

unstructured_ingest/runner/outlook.py

@@ -1,33 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.outlook import SimpleOutlookConfig
-
-
-@dataclass
-class OutlookRunner(Runner):
-    connector_config: "SimpleOutlookConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(self.connector_config.user_email.encode("utf-8"))
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="outlook",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.outlook import (
-            OutlookSourceConnector,
-        )
-
-        return OutlookSourceConnector

unstructured_ingest/runner/reddit.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.reddit import SimpleRedditConfig
-
-
-@dataclass
-class RedditRunner(Runner):
-    connector_config: "SimpleRedditConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            self.connector_config.subreddit_name.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="reddit",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.reddit import (
-            RedditSourceConnector,
-        )
-
-        return RedditSourceConnector

unstructured_ingest/runner/salesforce.py

@@ -1,33 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.salesforce import SimpleSalesforceConfig
-
-
-@dataclass
-class SalesforceRunner(Runner):
-    connector_config: "SimpleSalesforceConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(self.connector_config.username.encode("utf-8"))
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="salesforce",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.salesforce import (
-            SalesforceSourceConnector,
-        )
-
-        return SalesforceSourceConnector

unstructured_ingest/runner/sharepoint.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.sharepoint import SimpleSharepointConfig
-
-
-@dataclass
-class SharePointRunner(Runner):
-    connector_config: "SimpleSharepointConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            f"{self.connector_config.site}_{self.connector_config.path}".encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="sharepoint",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.sharepoint import (
-            SharepointSourceConnector,
-        )
-
-        return SharepointSourceConnector

unstructured_ingest/runner/slack.py

@@ -1,33 +0,0 @@
-import hashlib
-import typing as t
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.slack import SimpleSlackConfig
-
-
-class SlackRunner(Runner):
-    connector_config: "SimpleSlackConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            ",".join(self.connector_config.channels).encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="slack",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.slack import (
-            SlackSourceConnector,
-        )
-
-        return SlackSourceConnector

unstructured_ingest/runner/utils.py

@@ -1,47 +0,0 @@
-from __future__ import annotations
-
-import hashlib
-import logging
-from pathlib import Path
-
-from unstructured_ingest.interfaces import (
-    ReadConfig,
-)
-
-
-def update_download_dir_remote_url(
-    connector_name: str,
-    read_config: ReadConfig,
-    remote_url: str,
-    logger: logging.Logger,
-) -> str:
-    hashed_dir_name = hashlib.sha256(remote_url.encode("utf-8"))
-    return update_download_dir_hash(
-        connector_name=connector_name,
-        read_config=read_config,
-        hashed_dir_name=hashed_dir_name,
-        logger=logger,
-    )
-
-
-def update_download_dir_hash(
-    connector_name: str,
-    read_config: ReadConfig,
-    hashed_dir_name: hashlib._Hash,
-    logger: logging.Logger,
-) -> str:
-    if not read_config.download_dir:
-        cache_path = Path.home() / ".cache" / "unstructured" / "ingest"
-        if not cache_path.exists():
-            cache_path.mkdir(parents=True, exist_ok=True)
-        download_dir = cache_path / connector_name / hashed_dir_name.hexdigest()[:10]
-        if read_config.preserve_downloads:
-            logger.warning(
-                f"Preserving downloaded files but download_dir is not specified,"
-                f" using {download_dir}",
-            )
-        new_download_dir = str(download_dir)
-        logger.debug(f"updating download directory to: {new_download_dir}")
-    else:
-        new_download_dir = read_config.download_dir
-    return new_download_dir

unstructured_ingest/runner/wikipedia.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.wikipedia import SimpleWikipediaConfig
-
-
-@dataclass
-class WikipediaRunner(Runner):
-    connector_config: "SimpleWikipediaConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            self.connector_config.page_title.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="wikipedia",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.wikipedia import (
-            WikipediaSourceConnector,
-        )
-
-        return WikipediaSourceConnector

unstructured_ingest/runner/writers/__init__.py

@@ -1,48 +0,0 @@
-import typing as t
-
-from .astradb import AstraDBWriter
-from .azure_ai_search import AzureAiSearchWriter
-from .base_writer import Writer
-from .chroma import ChromaWriter
-from .clarifai import ClarifaiWriter
-from .databricks_volumes import DatabricksVolumesWriter
-from .delta_table import DeltaTableWriter
-from .elasticsearch import ElasticsearchWriter
-from .fsspec.azure import AzureWriter
-from .fsspec.box import BoxWriter
-from .fsspec.dropbox import DropboxWriter
-from .fsspec.gcs import GcsWriter
-from .fsspec.s3 import S3Writer
-from .kafka import KafkaWriter
-from .mongodb import MongodbWriter
-from .opensearch import OpenSearchWriter
-from .pinecone import PineconeWriter
-from .qdrant import QdrantWriter
-from .sql import SqlWriter
-from .vectara import VectaraWriter
-from .weaviate import WeaviateWriter
-
-writer_map: t.Dict[str, t.Type[Writer]] = {
-    "astradb": AstraDBWriter,
-    "azure": AzureWriter,
-    "azure_ai_search": AzureAiSearchWriter,
-    "box": BoxWriter,
-    "chroma": ChromaWriter,
-    "clarifai": ClarifaiWriter,
-    "databricks_volumes": DatabricksVolumesWriter,
-    "delta_table": DeltaTableWriter,
-    "dropbox": DropboxWriter,
-    "elasticsearch": ElasticsearchWriter,
-    "gcs": GcsWriter,
-    "kafka": KafkaWriter,
-    "mongodb": MongodbWriter,
-    "opensearch": OpenSearchWriter,
-    "pinecone": PineconeWriter,
-    "qdrant": QdrantWriter,
-    "s3": S3Writer,
-    "sql": SqlWriter,
-    "vectara": VectaraWriter,
-    "weaviate": WeaviateWriter,
-}
-
-__all__ = ["writer_map"]

unstructured_ingest/runner/writers/astradb.py

@@ -1,22 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.interfaces import BaseDestinationConnector
-from unstructured_ingest.runner.writers.base_writer import Writer
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.astradb import AstraDBWriteConfig, SimpleAstraDBConfig
-
-
-@dataclass
-class AstraDBWriter(Writer, EnhancedDataClassJsonMixin):
-    write_config: "AstraDBWriteConfig"
-    connector_config: "SimpleAstraDBConfig"
-
-    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
-        from unstructured_ingest.connector.astradb import (
-            AstraDBDestinationConnector,
-        )
-
-        return AstraDBDestinationConnector

unstructured_ingest/runner/writers/azure_ai_search.py

@@ -1,24 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseDestinationConnector
-from unstructured_ingest.runner.writers.base_writer import Writer
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.azure_ai_search import (
-        AzureAISearchWriteConfig,
-        SimpleAzureAISearchStorageConfig,
-    )
-
-
-@dataclass
-class AzureAiSearchWriter(Writer):
-    connector_config: "SimpleAzureAISearchStorageConfig"
-    write_config: "AzureAISearchWriteConfig"
-
-    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
-        from unstructured_ingest.connector.azure_ai_search import (
-            AzureAISearchDestinationConnector,
-        )
-
-        return AzureAISearchDestinationConnector

unstructured_ingest/runner/writers/base_writer.py

@@ -1,26 +0,0 @@
-import typing as t
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import (
-    BaseConnectorConfig,
-    BaseDestinationConnector,
-    WriteConfig,
-)
-
-
-@dataclass
-class Writer(ABC):
-    connector_config: BaseConnectorConfig
-    write_config: WriteConfig
-
-    @abstractmethod
-    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
-        pass
-
-    def get_connector(self, **kwargs) -> BaseDestinationConnector:
-        connector_cls = self.get_connector_cls()
-        return connector_cls(
-            write_config=self.write_config,
-            connector_config=self.connector_config,
-        )

unstructured_ingest/runner/writers/chroma.py

@@ -1,22 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.interfaces import BaseDestinationConnector
-from unstructured_ingest.runner.writers.base_writer import Writer
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.chroma import ChromaWriteConfig, SimpleChromaConfig
-
-
-@dataclass
-class ChromaWriter(Writer, EnhancedDataClassJsonMixin):
-    write_config: "ChromaWriteConfig"
-    connector_config: "SimpleChromaConfig"
-
-    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
-        from unstructured_ingest.connector.chroma import (
-            ChromaDestinationConnector,
-        )
-
-        return ChromaDestinationConnector

unstructured_ingest/runner/writers/clarifai.py

@@ -1,19 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseDestinationConnector
-from unstructured_ingest.runner.writers.base_writer import Writer
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.clarifai import ClarifaiWriteConfig, SimpleClarifaiConfig
-
-
-@dataclass
-class ClarifaiWriter(Writer):
-    write_config: "ClarifaiWriteConfig"
-    connector_config: "SimpleClarifaiConfig"
-
-    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
-        from unstructured_ingest.connector.clarifai import ClarifaiDestinationConnector
-
-        return ClarifaiDestinationConnector