unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/runner/base_runner.py

@@ -1,89 +0,0 @@
-import logging
-import typing as t
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.interfaces import (
-    BaseConnectorConfig,
-    BaseDestinationConnector,
-    BaseSourceConnector,
-    ChunkingConfig,
-    EmbeddingConfig,
-    PartitionConfig,
-    PermissionsConfig,
-    ProcessorConfig,
-    ReadConfig,
-    RetryStrategyConfig,
-)
-from unstructured_ingest.logger import ingest_log_streaming_init
-from unstructured_ingest.processor import process_documents
-from unstructured_ingest.runner.writers.base_writer import Writer
-
-
-@dataclass
-class Runner(EnhancedDataClassJsonMixin, ABC):
-    connector_config: BaseConnectorConfig
-    processor_config: ProcessorConfig
-    read_config: ReadConfig
-    partition_config: PartitionConfig
-    writer: t.Optional[Writer] = None
-    writer_kwargs: t.Optional[dict] = None
-    embedding_config: t.Optional[EmbeddingConfig] = None
-    chunking_config: t.Optional[ChunkingConfig] = None
-    permissions_config: t.Optional[PermissionsConfig] = None
-    retry_strategy_config: t.Optional[RetryStrategyConfig] = None
-
-    def run(self, *args, **kwargs):
-        ingest_log_streaming_init(logging.DEBUG if self.processor_config.verbose else logging.INFO)
-        self.update_read_config()
-        source_connector = self.get_source_connector()
-        self.process_documents(
-            source_doc_connector=source_connector,
-        )
-
-    @abstractmethod
-    def update_read_config(self):
-        pass
-
-    @abstractmethod
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        pass
-
-    def get_source_connector(self) -> BaseSourceConnector:
-        source_connector_cls = self.get_source_connector_cls()
-        return source_connector_cls(
-            processor_config=self.processor_config,
-            connector_config=self.connector_config,
-            read_config=self.read_config,
-        )
-
-    def get_dest_doc_connector(self) -> t.Optional[BaseDestinationConnector]:
-        writer_kwargs = self.writer_kwargs if self.writer_kwargs else {}
-        if self.writer:
-            return self.writer.get_connector(**writer_kwargs)
-        return None
-
-    def get_permissions_config(self) -> t.Optional[PermissionsConfig]:
-        if self.permissions_config is None:
-            return None
-
-        permissions_config_filled = bool(
-            self.permissions_config.application_id
-            and self.permissions_config.client_cred
-            and self.permissions_config.tenant,
-        )
-
-        return self.permissions_config if permissions_config_filled else None
-
-    def process_documents(self, source_doc_connector: BaseSourceConnector):
-        process_documents(
-            processor_config=self.processor_config,
-            source_doc_connector=source_doc_connector,
-            partition_config=self.partition_config,
-            dest_doc_connector=self.get_dest_doc_connector(),
-            embedder_config=self.embedding_config,
-            chunking_config=self.chunking_config,
-            permissions_config=self.get_permissions_config(),
-            retry_strategy_config=self.retry_strategy_config,
-        )

unstructured_ingest/runner/biomed.py

@@ -1,45 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.biomed import SimpleBiomedConfig
-
-
-@dataclass
-class BiomedRunner(Runner):
-    connector_config: "SimpleBiomedConfig"
-
-    def update_read_config(self):
-        base_path = (
-            self.connector_config.path
-            if self.connector_config.path
-            else "{}-{}-{}".format(
-                self.connector_config.api_id if self.connector_config.api_id else "",
-                self.connector_config.api_from if self.connector_config.api_from else "",
-                self.connector_config.api_until if self.connector_config.api_until else "",
-            )
-        )
-
-        hashed_dir_name = hashlib.sha256(
-            base_path.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="biomed",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.biomed import (
-            BiomedSourceConnector,
-        )
-
-        return BiomedSourceConnector

unstructured_ingest/runner/confluence.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.confluence import SimpleConfluenceConfig
-
-
-@dataclass
-class ConfluenceRunner(Runner):
-    connector_config: "SimpleConfluenceConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            self.connector_config.url.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="confluence",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.confluence import (
-            ConfluenceSourceConnector,
-        )
-
-        return ConfluenceSourceConnector

unstructured_ingest/runner/delta_table.py

@@ -1,34 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.delta_table import SimpleDeltaTableConfig
-
-
-@dataclass
-class DeltaTableRunner(Runner):
-    connector_config: "SimpleDeltaTableConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            str(self.connector_config.table_uri).encode("utf-8"),
-        )
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="delta_table",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.delta_table import (
-            DeltaTableSourceConnector,
-        )
-
-        return DeltaTableSourceConnector

unstructured_ingest/runner/discord.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.discord import SimpleDiscordConfig
-
-
-@dataclass
-class DiscordRunner(Runner):
-    connector_config: "SimpleDiscordConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            ",".join(self.connector_config.channels).encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="discord",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.discord import (
-            DiscordSourceConnector,
-        )
-
-        return DiscordSourceConnector

unstructured_ingest/runner/elasticsearch.py

@@ -1,40 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.elasticsearch import SimpleElasticsearchConfig
-
-
-@dataclass
-class ElasticSearchRunner(Runner):
-    connector_config: "SimpleElasticsearchConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            "{}_{}".format(
-                ",".join(self.connector_config.access_config.hosts),
-                self.connector_config.index_name,
-            ).encode(
-                "utf-8",
-            ),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="elasticsearch",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.elasticsearch import (
-            ElasticsearchSourceConnector,
-        )
-
-        return ElasticsearchSourceConnector

unstructured_ingest/runner/fsspec/azure.py

@@ -1,30 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.azure import SimpleAzureBlobStorageConfig
-
-
-@dataclass
-class AzureRunner(Runner):
-    connector_config: "SimpleAzureBlobStorageConfig"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="azure",
-            read_config=self.read_config,
-            remote_url=self.connector_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.azure import (
-            AzureBlobStorageSourceConnector,
-        )
-
-        return AzureBlobStorageSourceConnector

unstructured_ingest/runner/fsspec/box.py

@@ -1,28 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.box import SimpleBoxConfig
-
-
-@dataclass
-class BoxRunner(Runner):
-    connector_config: "SimpleBoxConfig"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="box",
-            read_config=self.read_config,
-            remote_url=self.connector_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.box import BoxSourceConnector
-
-        return BoxSourceConnector

unstructured_ingest/runner/fsspec/dropbox.py

@@ -1,30 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.dropbox import SimpleDropboxConfig
-
-
-@dataclass
-class DropboxRunner(Runner):
-    connector_config: "SimpleDropboxConfig"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="dropbox",
-            read_config=self.read_config,
-            remote_url=self.connector_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.dropbox import (
-            DropboxSourceConnector,
-        )
-
-        return DropboxSourceConnector

unstructured_ingest/runner/fsspec/fsspec.py

@@ -1,40 +0,0 @@
-import typing as t
-import warnings
-from dataclasses import dataclass
-from urllib.parse import urlparse
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.fsspec import SimpleFsspecConfig
-
-
-@dataclass
-class FsspecRunner(Runner):
-    connector_config: "SimpleFsspecConfig"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="fsspec",
-            read_config=self.read_config,
-            remote_url=self.fsspec_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-        protocol = urlparse(self.fsspec_config.remote_url).scheme  # type: ignore
-        warnings.warn(
-            f"`fsspec` protocol {protocol} is not directly supported by `unstructured`,"
-            " so use it at your own risk. Supported protocols are `gcs`, `gs`, `s3`, `s3a`,"
-            "`dropbox`, `abfs`, `az` and `sftp`.",
-            UserWarning,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.fsspec import (
-            FsspecSourceConnector,
-        )
-
-        return FsspecSourceConnector

unstructured_ingest/runner/fsspec/gcs.py

@@ -1,28 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.gcs import SimpleGcsConfig
-
-
-@dataclass
-class GCSRunner(Runner):
-    connector_config: "SimpleGcsConfig"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="gcs",
-            read_config=self.read_config,
-            remote_url=self.connector_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.gcs import GcsSourceConnector
-
-        return GcsSourceConnector

unstructured_ingest/runner/fsspec/s3.py

@@ -1,28 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.s3 import SimpleS3Config
-
-
-@dataclass
-class S3Runner(Runner):
-    connector_config: "SimpleS3Config"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="s3",
-            read_config=self.read_config,
-            remote_url=self.connector_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.s3 import S3SourceConnector
-
-        return S3SourceConnector

unstructured_ingest/runner/fsspec/sftp.py

@@ -1,28 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_remote_url
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.fsspec.sftp import SimpleSftpConfig
-
-
-@dataclass
-class SftpRunner(Runner):
-    connector_config: "SimpleSftpConfig"
-
-    def update_read_config(self):
-        self.read_config.download_dir = update_download_dir_remote_url(
-            connector_name="sftp",
-            read_config=self.read_config,
-            remote_url=self.connector_config.remote_url,  # type: ignore
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.fsspec.sftp import SftpSourceConnector
-
-        return SftpSourceConnector

unstructured_ingest/runner/github.py

@@ -1,37 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.github import SimpleGitHubConfig
-
-
-@dataclass
-class GithubRunner(Runner):
-    connector_config: "SimpleGitHubConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            f"{self.connector_config.url}_{self.connector_config.branch}".encode(
-                "utf-8",
-            ),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="github",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.github import (
-            GitHubSourceConnector,
-        )
-
-        return GitHubSourceConnector

unstructured_ingest/runner/gitlab.py

@@ -1,37 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.gitlab import SimpleGitlabConfig
-
-
-@dataclass
-class GitlabRunner(Runner):
-    connector_config: "SimpleGitlabConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            f"{self.connector_config.url}_{self.connector_config.branch}".encode(
-                "utf-8",
-            ),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="gitlab",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.gitlab import (
-            GitLabSourceConnector,
-        )
-
-        return GitLabSourceConnector

unstructured_ingest/runner/google_drive.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.google_drive import SimpleGoogleDriveConfig
-
-
-@dataclass
-class GoogleDriveRunner(Runner):
-    connector_config: "SimpleGoogleDriveConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            self.connector_config.drive_id.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="google_drive",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.google_drive import (
-            GoogleDriveSourceConnector,
-        )
-
-        return GoogleDriveSourceConnector

unstructured_ingest/runner/hubspot.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.hubspot import SimpleHubSpotConfig
-
-
-@dataclass
-class HubSpotRunner(Runner):
-    connector_config: "SimpleHubSpotConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            self.connector_config.access_config.api_token.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="hubspot",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.hubspot import (
-            HubSpotSourceConnector,
-        )
-
-        return HubSpotSourceConnector

unstructured_ingest/runner/jira.py

@@ -1,35 +0,0 @@
-import hashlib
-import typing as t
-from dataclasses import dataclass
-
-from unstructured_ingest.interfaces import BaseSourceConnector
-from unstructured_ingest.logger import logger
-from unstructured_ingest.runner.base_runner import Runner
-from unstructured_ingest.runner.utils import update_download_dir_hash
-
-if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.jira import SimpleJiraConfig
-
-
-@dataclass
-class JiraRunner(Runner):
-    connector_config: "SimpleJiraConfig"
-
-    def update_read_config(self):
-        hashed_dir_name = hashlib.sha256(
-            self.connector_config.url.encode("utf-8"),
-        )
-
-        self.read_config.download_dir = update_download_dir_hash(
-            connector_name="jira",
-            read_config=self.read_config,
-            hashed_dir_name=hashed_dir_name,
-            logger=logger,
-        )
-
-    def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.jira import (
-            JiraSourceConnector,
-        )
-
-        return JiraSourceConnector