unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/runner/writers/fsspec/azure.py

@@ -0,0 +1,24 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.fsspec.azure import (
+        AzureWriteConfig,
+        SimpleAzureBlobStorageConfig,
+    )
+
+
+@dataclass
+class AzureWriter(Writer):
+    connector_config: "SimpleAzureBlobStorageConfig"
+    write_config: "AzureWriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.fsspec.azure import (
+            AzureBlobStorageDestinationConnector,
+        )
+
+        return AzureBlobStorageDestinationConnector

unstructured_ingest/runner/writers/fsspec/box.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.fsspec.box import BoxWriteConfig, SimpleBoxConfig
+
+
+@dataclass
+class BoxWriter(Writer):
+    connector_config: "SimpleBoxConfig"
+    write_config: "BoxWriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.fsspec.box import (
+            BoxDestinationConnector,
+        )
+
+        return BoxDestinationConnector

unstructured_ingest/runner/writers/fsspec/dropbox.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.fsspec.dropbox import DropboxWriteConfig, SimpleDropboxConfig
+
+
+@dataclass
+class DropboxWriter(Writer):
+    connector_config: "SimpleDropboxConfig"
+    write_config: "DropboxWriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.fsspec.dropbox import (
+            DropboxDestinationConnector,
+        )
+
+        return DropboxDestinationConnector

unstructured_ingest/runner/writers/fsspec/gcs.py

@@ -0,0 +1,19 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.fsspec.gcs import GcsWriteConfig, SimpleGcsConfig
+
+
+@dataclass
+class GcsWriter(Writer):
+    connector_config: "SimpleGcsConfig"
+    write_config: "GcsWriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.fsspec.gcs import GcsDestinationConnector
+
+        return GcsDestinationConnector

unstructured_ingest/runner/writers/fsspec/s3.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.fsspec.s3 import S3WriteConfig, SimpleS3Config
+
+
+@dataclass
+class S3Writer(Writer):
+    connector_config: "SimpleS3Config"
+    write_config: "S3WriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.fsspec.s3 import (
+            S3DestinationConnector,
+        )
+
+        return S3DestinationConnector

unstructured_ingest/runner/writers/kafka.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.kafka import KafkaWriteConfig, SimpleKafkaConfig
+
+
+@dataclass
+class KafkaWriter(Writer):
+    write_config: "KafkaWriteConfig"
+    connector_config: "SimpleKafkaConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.kafka import (
+            KafkaDestinationConnector,
+        )
+
+        return KafkaDestinationConnector

unstructured_ingest/runner/writers/mongodb.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.mongodb import MongoDBWriteConfig, SimpleMongoDBConfig
+
+
+@dataclass
+class MongodbWriter(Writer):
+    write_config: "MongoDBWriteConfig"
+    connector_config: "SimpleMongoDBConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.mongodb import (
+            MongoDBDestinationConnector,
+        )
+
+        return MongoDBDestinationConnector

unstructured_ingest/runner/writers/opensearch.py

@@ -0,0 +1,26 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.elasticsearch import (
+        ElasticsearchWriteConfig,
+    )
+    from unstructured_ingest.connector.opensearch import (
+        SimpleOpenSearchConfig,
+    )
+
+
+@dataclass
+class OpenSearchWriter(Writer):
+    connector_config: "SimpleOpenSearchConfig"
+    write_config: "ElasticsearchWriteConfig"
+
+    def get_connector_cls(self) -> BaseDestinationConnector:
+        from unstructured_ingest.connector.opensearch import (
+            OpenSearchDestinationConnector,
+        )
+
+        return OpenSearchDestinationConnector

unstructured_ingest/runner/writers/pinecone.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.pinecone import PineconeWriteConfig, SimplePineconeConfig
+
+
+@dataclass
+class PineconeWriter(Writer):
+    write_config: "PineconeWriteConfig"
+    connector_config: "SimplePineconeConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.pinecone import (
+            PineconeDestinationConnector,
+        )
+
+        return PineconeDestinationConnector

unstructured_ingest/runner/writers/qdrant.py

@@ -0,0 +1,19 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.qdrant import QdrantWriteConfig, SimpleQdrantConfig
+
+
+@dataclass
+class QdrantWriter(Writer):
+    write_config: "QdrantWriteConfig"
+    connector_config: "SimpleQdrantConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.qdrant import QdrantDestinationConnector
+
+        return QdrantDestinationConnector

unstructured_ingest/runner/writers/sql.py

@@ -0,0 +1,22 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.sql import SimpleSqlConfig
+    from unstructured_ingest.interfaces import WriteConfig
+
+
+@dataclass
+class SqlWriter(Writer):
+    write_config: "WriteConfig"
+    connector_config: "SimpleSqlConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.sql import (
+            SqlDestinationConnector,
+        )
+
+        return SqlDestinationConnector

unstructured_ingest/runner/writers/vectara.py

@@ -0,0 +1,22 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.vectara import SimpleVectaraConfig, VectaraWriteConfig
+
+
+@dataclass
+class VectaraWriter(Writer, EnhancedDataClassJsonMixin):
+    write_config: "VectaraWriteConfig"
+    connector_config: "SimpleVectaraConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.vectara import (
+            VectaraDestinationConnector,
+        )
+
+        return VectaraDestinationConnector

unstructured_ingest/runner/writers/weaviate.py

@@ -0,0 +1,21 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.weaviate import SimpleWeaviateConfig, WeaviateWriteConfig
+
+
+@dataclass
+class WeaviateWriter(Writer):
+    write_config: "WeaviateWriteConfig"
+    connector_config: "SimpleWeaviateConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.weaviate import (
+            WeaviateDestinationConnector,
+        )
+
+        return WeaviateDestinationConnector

unstructured_ingest/utils/compression.py

@@ -0,0 +1,117 @@
+import copy
+import os
+import sys
+import tarfile
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional
+
+from unstructured_ingest.connector.local import LocalSourceConnector, SimpleLocalConfig
+from unstructured_ingest.interfaces import (
+    BaseConnectorConfig,
+    BaseSingleIngestDoc,
+    ProcessorConfig,
+    ReadConfig,
+)
+from unstructured_ingest.logger import logger
+
+ZIP_FILE_EXT = [".zip"]
+TAR_FILE_EXT = [".tar", ".tar.gz", ".tgz"]
+
+
+def uncompress_file(filename: str, path: Optional[str] = None) -> str:
+    """
+    Takes in a compressed zip or tar file and uncompresses it
+    """
+    # Create path if it doesn't already exist
+    if path:
+        Path(path).mkdir(parents=True, exist_ok=True)
+
+    if any(filename.endswith(ext) for ext in ZIP_FILE_EXT):
+        return uncompress_zip_file(zip_filename=filename, path=path)
+    elif any(filename.endswith(ext) for ext in TAR_FILE_EXT):
+        return uncompress_tar_file(tar_filename=filename, path=path)
+    else:
+        raise ValueError(
+            "filename {} not a recognized compressed extension: {}".format(
+                filename,
+                ", ".join(ZIP_FILE_EXT + TAR_FILE_EXT),
+            ),
+        )
+
+
+def uncompress_zip_file(zip_filename: str, path: Optional[str] = None) -> str:
+    head, tail = os.path.split(zip_filename)
+    for ext in ZIP_FILE_EXT:
+        if tail.endswith(ext):
+            tail = tail[: -(len(ext))]
+            break
+    path = path if path else os.path.join(head, f"{tail}-zip-uncompressed")
+    logger.info(f"extracting zip {zip_filename} -> {path}")
+    with zipfile.ZipFile(zip_filename) as zfile:
+        zfile.extractall(path=path)
+    return path
+
+
+def uncompress_tar_file(tar_filename: str, path: Optional[str] = None) -> str:
+    head, tail = os.path.split(tar_filename)
+    for ext in TAR_FILE_EXT:
+        if tail.endswith(ext):
+            tail = tail[: -(len(ext))]
+            break
+
+    path = path if path else os.path.join(head, f"{tail}-tar-uncompressed")
+    logger.info(f"extracting tar {tar_filename} -> {path}")
+    with tarfile.open(tar_filename, "r:gz") as tfile:
+        # NOTE(robinson: Mitigate against malicious content being extracted from the tar file.
+        # This was added in Python 3.12
+        # Ref: https://docs.python.org/3/library/tarfile.html#extraction-filters
+        if sys.version_info >= (3, 12):
+            tfile.extraction_filter = tarfile.tar_filter
+        else:
+            logger.warning(
+                "Extraction filtering for tar files is available for Python 3.12 and above. "
+                "Consider upgrading your Python version to improve security. "
+                "See https://docs.python.org/3/library/tarfile.html#extraction-filters"
+            )
+        tfile.extractall(path=path)
+    return path
+
+
+@dataclass
+class CompressionSourceConnectorMixin:
+    processor_config: ProcessorConfig
+    read_config: ReadConfig
+    connector_config: BaseConnectorConfig
+
+    def process_compressed_doc(self, doc: BaseSingleIngestDoc) -> List[BaseSingleIngestDoc]:
+        """
+        Utility function which helps process compressed files. Extracts the contents and returns
+        generated ingest docs via local source connector
+        """
+        # Download the raw file to local
+        doc.get_file()
+        path = uncompress_file(filename=str(doc.filename))
+        new_read_configs = copy.copy(self.read_config)
+        new_process_configs = copy.copy(self.processor_config)
+        relative_path = path.replace(self.read_config.download_dir, "")
+
+        if self.processor_config.output_dir.endswith(os.sep):
+            new_process_configs.output_dir = f"{self.processor_config.output_dir}{relative_path}"
+        else:
+            new_process_configs.output_dir = (
+                f"{self.processor_config.output_dir}{os.sep}{relative_path}"
+            )
+
+        local_connector = LocalSourceConnector(
+            connector_config=SimpleLocalConfig(
+                input_path=path,
+                recursive=True,
+            ),
+            read_config=new_read_configs,
+            processor_config=new_process_configs,
+        )
+        logger.info(f"Created local source connector: {local_connector.to_json()}")
+        local_connector.initialize()
+        return local_connector.get_ingest_docs()

unstructured_ingest/utils/data_prep.py

@@ -0,0 +1,112 @@
+import itertools
+import json
+from datetime import datetime
+from typing import Any, Optional, Sequence, cast
+
+DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
+
+
+def batch_generator(iterable, batch_size=100):
+    """A helper function to break an iterable into batches of size batch_size."""
+    it = iter(iterable)
+    chunk = tuple(itertools.islice(it, batch_size))
+    while chunk:
+        yield chunk
+        chunk = tuple(itertools.islice(it, batch_size))
+
+
+def generator_batching_wbytes(iterable, batch_size_limit_bytes=15_000_000):
+    """A helper function to break an iterable into chunks of specified bytes."""
+    current_batch, current_batch_size = [], 0
+
+    for item in iterable:
+        item_size_bytes = len(json.dumps(item).encode("utf-8"))
+
+        if current_batch_size + item_size_bytes <= batch_size_limit_bytes:
+            current_batch.append(item)
+            current_batch_size += item_size_bytes
+        else:
+            yield current_batch
+            current_batch, current_batch_size = [item], item_size_bytes
+
+    if current_batch:
+        yield current_batch
+
+
+def flatten_dict(
+    dictionary: dict[str, Any],
+    parent_key: str = "",
+    separator: str = "_",
+    flatten_lists: bool = False,
+    remove_none: bool = False,
+    keys_to_omit: Optional[Sequence[str]] = None,
+) -> dict[str, Any]:
+    """Flattens a nested dictionary into a single level dictionary.
+
+    keys_to_omit is a list of keys that don't get flattened. If omitting a nested key, format as
+    {parent_key}{separator}{key}. If flatten_lists is True, then lists and tuples are flattened as
+    well. If remove_none is True, then None keys/values are removed from the flattened
+    dictionary.
+    """
+    keys_to_omit = keys_to_omit if keys_to_omit else []
+    flattened_dict: dict[str, Any] = {}
+    for key, value in dictionary.items():
+        new_key = f"{parent_key}{separator}{key}" if parent_key else key
+        if new_key in keys_to_omit:
+            flattened_dict[new_key] = value
+        elif value is None and remove_none:
+            continue
+        elif isinstance(value, dict):
+            value = cast("dict[str, Any]", value)
+            flattened_dict.update(
+                flatten_dict(
+                    value, new_key, separator, flatten_lists, remove_none, keys_to_omit=keys_to_omit
+                ),
+            )
+        elif isinstance(value, (list, tuple)) and flatten_lists:
+            value = cast("list[Any] | tuple[Any]", value)
+            for index, item in enumerate(value):
+                flattened_dict.update(
+                    flatten_dict(
+                        {f"{new_key}{separator}{index}": item},
+                        "",
+                        separator,
+                        flatten_lists,
+                        remove_none,
+                        keys_to_omit=keys_to_omit,
+                    )
+                )
+        else:
+            flattened_dict[new_key] = value
+
+    return flattened_dict
+
+
+def validate_date_args(date: Optional[str] = None) -> bool:
+    """Validate whether the provided date string satisfies any of the supported date formats.
+
+    Used by unstructured/ingest/connector/biomed.py
+
+    Returns `True` if the date string satisfies any of the supported formats, otherwise raises
+    `ValueError`.
+
+    Supported Date Formats:
+        - 'YYYY-MM-DD'
+        - 'YYYY-MM-DDTHH:MM:SS'
+        - 'YYYY-MM-DD+HH:MM:SS'
+        - 'YYYY-MM-DDTHH:MM:SS±HHMM'
+    """
+    if not date:
+        raise ValueError("The argument date is None.")
+
+    for format in DATE_FORMATS:
+        try:
+            datetime.strptime(date, format)
+            return True
+        except ValueError:
+            pass
+
+    raise ValueError(
+        f"The argument {date} does not satisfy the format:"
+        f" YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or YYYY-MM-DD+HH:MM:SS or YYYY-MM-DDTHH:MM:SS±HHMM",
+    )

unstructured_ingest/utils/dep_check.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+from functools import wraps
+from typing import (
+    Callable,
+    List,
+    Optional,
+    TypeVar,
+)
+
+from typing_extensions import ParamSpec
+
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+
+def requires_dependencies(
+    dependencies: str | list[str],
+    extras: Optional[str] = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    if isinstance(dependencies, str):
+        dependencies = [dependencies]
+
+    def decorator(func: Callable[_P, _T]) -> Callable[_P, _T]:
+        def run_check():
+            missing_deps: List[str] = []
+            for dep in dependencies:
+                if not dependency_exists(dep):
+                    missing_deps.append(dep)
+            if len(missing_deps) > 0:
+                raise ImportError(
+                    f"Following dependencies are missing: {', '.join(missing_deps)}. "
+                    + (
+                        f"""Please install them using `pip install "unstructured[{extras}]"`."""
+                        if extras
+                        else f"Please install them using `pip install {' '.join(missing_deps)}`."
+                    ),
+                )
+
+        @wraps(func)
+        def wrapper(*args: _P.args, **kwargs: _P.kwargs):
+            run_check()
+            return func(*args, **kwargs)
+
+        @wraps(func)
+        async def wrapper_async(*args: _P.args, **kwargs: _P.kwargs):
+            run_check()
+            return await func(*args, **kwargs)
+
+        if asyncio.iscoroutinefunction(func):
+            return wrapper_async
+        return wrapper
+
+    return decorator
+
+
+def dependency_exists(dependency: str):
+    try:
+        importlib.import_module(dependency)
+    except ImportError as e:
+        # Check to make sure this isn't some unrelated import error.
+        if dependency in repr(e):
+            return False
+    return True

unstructured_ingest/utils/string_and_date_utils.py

@@ -0,0 +1,39 @@
+import json
+import typing as t
+from datetime import datetime
+
+from dateutil import parser
+
+
+def json_to_dict(json_string: str) -> t.Union[str, t.Dict[str, t.Any]]:
+    """Helper function attempts to deserialize json string to a dictionary."""
+    try:
+        return json.loads(json_string)
+    except json.JSONDecodeError:
+        # Not neccessary an error if it is a path or malformed json
+        pass
+    try:
+        # This is common when single quotes are used instead of double quotes
+        return json.loads(json_string.replace("'", '"'))
+    except json.JSONDecodeError:
+        # Not neccessary an error if it is a path
+        pass
+    return json_string
+
+
+def ensure_isoformat_datetime(timestamp: t.Union[datetime, str]) -> str:
+    """
+    Ensures that the input value is converted to an ISO format datetime string.
+    Handles both datetime objects and strings.
+    """
+    if isinstance(timestamp, datetime):
+        return timestamp.isoformat()
+    elif isinstance(timestamp, str):
+        try:
+            # Parse the datetime string in various formats
+            dt = parser.parse(timestamp)
+            return dt.isoformat()
+        except ValueError as e:
+            raise ValueError(f"String '{timestamp}' could not be parsed as a datetime.") from e
+    else:
+        raise TypeError(f"Expected input type datetime or str, but got {type(timestamp)}.")