unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/utils/dep_check.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import asyncio
+import importlib
+from functools import wraps
+from typing import (
+    Callable,
+    List,
+    Optional,
+    TypeVar,
+)
+
+from typing_extensions import ParamSpec
+
+_T = TypeVar("_T")
+_P = ParamSpec("_P")
+
+
+def requires_dependencies(
+    dependencies: str | list[str],
+    extras: Optional[str] = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    """Decorator ensuring required modules are installed.
+
+    Use on functions with local imports to ensure required modules are available and log
+    an installation instruction if they're not.
+
+    Args:
+        dependencies: Name(s) of module(s) required by the decorated function.
+        extras: unstructured-ingest extra which installs required `dependencies`. Defaults to None.
+
+    Raises:
+        ImportError: When at least one of the `dependencies` is not available.
+    """
+    if isinstance(dependencies, str):
+        dependencies = [dependencies]
+
+    def decorator(func: Callable[_P, _T]) -> Callable[_P, _T]:
+        def run_check():
+            missing_deps: List[str] = []
+            for dep in dependencies:
+                if not dependency_exists(dep):
+                    missing_deps.append(dep)
+            if len(missing_deps) > 0:
+                raise ImportError(
+                    f"Following dependencies are missing: {', '.join(missing_deps)}. "
+                    + (
+                        f"""Please install them using `pip install "unstructured-ingest[{extras}]"`."""  # noqa: E501
+                        if extras
+                        else f"Please install them using `pip install {' '.join(missing_deps)}`."
+                    ),
+                )
+
+        @wraps(func)
+        def wrapper(*args: _P.args, **kwargs: _P.kwargs):
+            run_check()
+            return func(*args, **kwargs)
+
+        @wraps(func)
+        async def wrapper_async(*args: _P.args, **kwargs: _P.kwargs):
+            run_check()
+            return await func(*args, **kwargs)
+
+        if asyncio.iscoroutinefunction(func):
+            return wrapper_async
+        return wrapper
+
+    return decorator
+
+
+def dependency_exists(dependency: str):
+    try:
+        importlib.import_module(dependency)
+    except ImportError as e:
+        # Check to make sure this isn't some unrelated import error.
+        if dependency in repr(e):
+            return False
+    return True

unstructured_ingest/utils/google_filetype.py

@@ -0,0 +1,9 @@
+GOOGLE_DRIVE_EXPORT_TYPES = {
+    "application/vnd.google-apps.document": "application/"
+    "vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.google-apps.spreadsheet": "application/"
+    "vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.google-apps.presentation": "application/"
+    "vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.google-apps.photo": "image/jpeg",
+}

unstructured_ingest/utils/string_and_date_utils.py

@@ -0,0 +1,49 @@
+import json
+import typing as t
+from datetime import datetime
+
+from dateutil import parser
+
+
+def json_to_dict(json_string: str) -> t.Union[str, t.Dict[str, t.Any]]:
+    """Helper function attempts to deserialize json string to a dictionary."""
+    try:
+        return json.loads(json_string)
+    except json.JSONDecodeError:
+        # Not necessary an error if it is a path or malformed json
+        pass
+    try:
+        # This is common when single quotes are used instead of double quotes
+        return json.loads(json_string.replace("'", '"'))
+    except json.JSONDecodeError:
+        # Not necessary an error if it is a path
+        pass
+    return json_string
+
+
+def ensure_isoformat_datetime(timestamp: t.Union[datetime, str]) -> str:
+    """
+    Ensures that the input value is converted to an ISO format datetime string.
+    Handles both datetime objects and strings.
+    """
+    if isinstance(timestamp, datetime):
+        return timestamp.isoformat()
+    elif isinstance(timestamp, str):
+        try:
+            # Parse the datetime string in various formats
+            dt = parser.parse(timestamp)
+            return dt.isoformat()
+        except ValueError as e:
+            raise ValueError(f"String '{timestamp}' could not be parsed as a datetime.") from e
+    else:
+        raise TypeError(f"Expected input type datetime or str, but got {type(timestamp)}.")
+
+
+def truncate_string_bytes(string: str, max_bytes: int, encoding: str = "utf-8") -> str:
+    """
+    Truncates a string to a specified maximum number of bytes.
+    """
+    encoded_string = str(string).encode(encoding)
+    if len(encoded_string) <= max_bytes:
+        return string
+    return encoded_string[:max_bytes].decode(encoding, errors="ignore")

unstructured_ingest/utils/table.py

@@ -0,0 +1,73 @@
+from typing import Any
+
+import pandas as pd
+
+from unstructured_ingest.utils.data_prep import flatten_dict
+
+
+def get_default_pandas_dtypes() -> dict[str, Any]:
+    return {
+        "text": pd.StringDtype(),  # type: ignore
+        "type": pd.StringDtype(),  # type: ignore
+        "element_id": pd.StringDtype(),  # type: ignore
+        "filename": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "filetype": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "file_directory": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "last_modified": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "attached_to_filename": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "parent_id": pd.StringDtype(),  # Optional[str],  # type: ignore
+        "category_depth": "Int64",  # Optional[int]
+        "image_path": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "languages": object,  # Optional[list[str]]
+        "page_number": "Int64",  # Optional[int]
+        "page_name": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "url": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "link_urls": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "link_texts": object,  # Optional[list[str]]
+        "links": object,
+        "sent_from": object,  # Optional[list[str]],
+        "sent_to": object,  # Optional[list[str]]
+        "subject": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "section": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "header_footer_type": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "emphasized_text_contents": object,  # Optional[list[str]]
+        "emphasized_text_tags": object,  # Optional[list[str]]
+        "text_as_html": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "regex_metadata": object,
+        "max_characters": "Int64",  # Optional[int]
+        "is_continuation": "boolean",  # Optional[bool]
+        "detection_class_prob": float,  # Optional[float],
+        "sender": pd.StringDtype(),  # type: ignore
+        "coordinates_points": object,
+        "coordinates_system": pd.StringDtype(),  # type: ignore
+        "coordinates_layout_width": float,
+        "coordinates_layout_height": float,
+        "data_source_url": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_version": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_record_locator": object,
+        "data_source_date_created": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_date_modified": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_date_processed": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_permissions_data": object,
+        "embeddings": object,
+        "regex_metadata_key": object,
+    }
+
+
+def convert_to_pandas_dataframe(
+    elements_dict: list[dict[str, Any]],
+    drop_empty_cols: bool = False,
+) -> pd.DataFrame:
+    # Flatten metadata if it hasn't already been flattened
+    for d in elements_dict:
+        if metadata := d.pop("metadata", None):
+            d.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
+
+    df = pd.DataFrame.from_dict(
+        elements_dict,
+    )
+    dt = {k: v for k, v in get_default_pandas_dtypes().items() if k in df.columns}
+    df = df.astype(dt)
+    if drop_empty_cols:
+        df.dropna(axis=1, how="all", inplace=True)
+    return df

unstructured_ingest/v2/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

unstructured_ingest/v2/cli/base/__init__.py

@@ -0,0 +1,4 @@
+from .dest import DestCmd
+from .src import SrcCmd
+
+__all__ = ["SrcCmd", "DestCmd"]

unstructured_ingest/v2/cli/base/cmd.py

@@ -0,0 +1,269 @@
+import inspect
+from abc import ABC, abstractmethod
+from collections import Counter
+from dataclasses import dataclass, field, fields
+from typing import Any, Optional, Type, TypeVar
+
+import click
+from pydantic import BaseModel
+
+from unstructured_ingest.v2.cli.base.importer import import_from_string
+from unstructured_ingest.v2.cli.utils.click import extract_config
+from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model, post_check
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.pipeline.pipeline import Pipeline
+from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
+from unstructured_ingest.v2.processes.connector_registry import (
+    DownloaderT,
+    IndexerT,
+    RegistryEntry,
+    UploaderT,
+    UploadStager,
+    UploadStagerConfig,
+    UploadStagerT,
+    destination_registry,
+    source_registry,
+)
+from unstructured_ingest.v2.processes.connectors.local import LocalUploader, LocalUploaderConfig
+from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.v2.processes.filter import Filterer, FiltererConfig
+from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
+
+CommandT = TypeVar("CommandT", bound=click.Command)
+
+
+@dataclass
+class BaseCmd(ABC):
+    cmd_name: str
+    registry_entry: RegistryEntry
+    default_configs: list[Type[BaseModel]] = field(default_factory=list)
+
+    @abstractmethod
+    def get_registry_options(self):
+        pass
+
+    def get_default_options(self) -> list[click.Option]:
+        options = []
+        for extra in self.default_configs:
+            options.extend(options_from_base_model(model=extra))
+        return options
+
+    @classmethod
+    def consolidate_options(cls, options: list[click.Option]) -> list[click.Option]:
+        option_names = [option.name for option in options]
+        duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+        if not duplicate_names:
+            return options
+        consolidated_options = []
+        current_names = []
+        for option in options:
+            if option.name not in current_names:
+                current_names.append(option.name)
+                consolidated_options.append(option)
+                continue
+            existing_option = next(o for o in consolidated_options if o.name == option.name)
+            if existing_option.__dict__ == option.__dict__:
+                continue
+            option_diff = cls.get_options_diff(o1=option, o2=existing_option)
+            raise ValueError(
+                "Conflicting duplicate {} option defined: {}".format(
+                    option.name, " | ".join([f"{d[0]}: {d[1]}" for d in option_diff])
+                )
+            )
+        return consolidated_options
+
+    @staticmethod
+    def get_options_diff(o1: click.Option, o2: click.Option):
+        o1_dict = o1.__dict__
+        o2_dict = o2.__dict__
+        for d in [o1_dict, o2_dict]:
+            d["opts"] = ",".join(d["opts"])
+            d["secondary_opts"] = ",".join(d["secondary_opts"])
+        option_diff = set(o1_dict.items()) ^ set(o2_dict.items())
+        return option_diff
+
+    @property
+    def cmd_name_key(self):
+        return self.cmd_name.replace("-", "_")
+
+    @property
+    def cli_cmd_name(self):
+        return self.cmd_name.replace("_", "-")
+
+    @abstractmethod
+    def cmd(self, ctx: click.Context, **options) -> None:
+        pass
+
+    def add_options(self, cmd: CommandT) -> CommandT:
+        options = self.get_registry_options()
+        options.extend(self.get_default_options())
+        post_check(options)
+        cmd.params.extend(options)
+        return cmd
+
+    def get_pipeline(
+        self,
+        src: str,
+        source_options: dict[str, Any],
+        dest: Optional[str] = None,
+        destination_options: Optional[dict[str, Any]] = None,
+    ) -> Pipeline:
+        logger.debug(
+            f"creating pipeline from cli using source {src} with options: {source_options}"
+        )
+        pipeline_kwargs: dict[str, Any] = {
+            "context": self.get_processor_config(options=source_options),
+            "downloader": self.get_downloader(src=src, options=source_options),
+            "indexer": self.get_indexer(src=src, options=source_options),
+            "partitioner": self.get_partitioner(options=source_options),
+        }
+        if chunker := self.get_chunker(options=source_options):
+            pipeline_kwargs["chunker"] = chunker
+        if filterer := self.get_filterer(options=source_options):
+            pipeline_kwargs["filterer"] = filterer
+        if embedder := self.get_embedder(options=source_options):
+            pipeline_kwargs["embedder"] = embedder
+        if dest:
+            logger.debug(
+                f"setting destination on pipeline {dest} with options: {destination_options}"
+            )
+            if uploader_stager := self.get_upload_stager(dest=dest, options=destination_options):
+                pipeline_kwargs["stager"] = uploader_stager
+            pipeline_kwargs["uploader"] = self.get_uploader(dest=dest, options=destination_options)
+        else:
+            # Default to local uploader
+            # TODO remove after v1 no longer supported
+            destination_options = destination_options or {}
+            if "output_dir" not in destination_options:
+                destination_options["output_dir"] = source_options["output_dir"]
+            pipeline_kwargs["uploader"] = self.get_default_uploader(options=destination_options)
+        return Pipeline(**pipeline_kwargs)
+
+    @staticmethod
+    def get_default_uploader(options: dict[str, Any]) -> UploaderT:
+        uploader_config = extract_config(flat_data=options, config=LocalUploaderConfig)
+        return LocalUploader(upload_config=uploader_config)
+
+    @staticmethod
+    def get_chunker(options: dict[str, Any]) -> Optional[Chunker]:
+        chunker_config = extract_config(flat_data=options, config=ChunkerConfig)
+        if not chunker_config.chunking_strategy:
+            return None
+        return Chunker(config=chunker_config)
+
+    @staticmethod
+    def get_filterer(options: dict[str, Any]) -> Optional[Filterer]:
+        filterer_configs = extract_config(flat_data=options, config=FiltererConfig)
+        if not filterer_configs.model_dump():
+            return None
+        return Filterer(config=filterer_configs)
+
+    @staticmethod
+    def get_embedder(options: dict[str, Any]) -> Optional[Embedder]:
+        embedder_config = extract_config(flat_data=options, config=EmbedderConfig)
+        if not embedder_config.embedding_provider:
+            return None
+        return Embedder(config=embedder_config)
+
+    @staticmethod
+    def get_partitioner(options: dict[str, Any]) -> Partitioner:
+        partitioner_config = extract_config(flat_data=options, config=PartitionerConfig)
+        return Partitioner(config=partitioner_config)
+
+    @staticmethod
+    def get_processor_config(options: dict[str, Any]) -> ProcessorConfig:
+        return extract_config(flat_data=options, config=ProcessorConfig)
+
+    @staticmethod
+    def get_indexer(src: str, options: dict[str, Any]) -> IndexerT:
+        source_entry = source_registry[src]
+        indexer_kwargs: dict[str, Any] = {}
+        if indexer_config_cls := source_entry.indexer_config:
+            indexer_kwargs["index_config"] = extract_config(
+                flat_data=options, config=indexer_config_cls
+            )
+        if connection_config_cls := source_entry.connection_config:
+            indexer_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        indexer_cls = source_entry.indexer
+        return indexer_cls(**indexer_kwargs)
+
+    @staticmethod
+    def get_downloader(src: str, options: dict[str, Any]) -> DownloaderT:
+        source_entry = source_registry[src]
+        downloader_kwargs: dict[str, Any] = {}
+        if downloader_config_cls := source_entry.downloader_config:
+            downloader_kwargs["download_config"] = extract_config(
+                flat_data=options, config=downloader_config_cls
+            )
+        if connection_config_cls := source_entry.connection_config:
+            downloader_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        downloader_cls = source_entry.downloader
+        return downloader_cls(**downloader_kwargs)
+
+    @staticmethod
+    def get_custom_stager(
+        stager_reference: str, stager_config_kwargs: Optional[dict] = None
+    ) -> Optional[UploadStagerT]:
+        uploader_cls = import_from_string(stager_reference)
+        if not inspect.isclass(uploader_cls):
+            raise ValueError(
+                f"custom stager must be a reference to a python class, got: {type(uploader_cls)}"
+            )
+        if not issubclass(uploader_cls, UploadStager):
+            raise ValueError(
+                "custom stager must be an implementation of the UploadStager interface"
+            )
+        fields_dict = {f.name: f.type for f in fields(uploader_cls)}
+        upload_stager_config_cls = fields_dict["upload_stager_config"]
+        if not inspect.isclass(upload_stager_config_cls):
+            raise ValueError(
+                f"custom stager config must be a class, got: {type(upload_stager_config_cls)}"
+            )
+        if not issubclass(upload_stager_config_cls, UploadStagerConfig):
+            raise ValueError(
+                "custom stager config must be an implementation "
+                "of the UploadStagerUploadStagerConfig interface"
+            )
+        upload_stager_kwargs: dict[str, Any] = {}
+        if stager_config_kwargs:
+            upload_stager_kwargs["upload_stager_config"] = upload_stager_config_cls(
+                **stager_config_kwargs
+            )
+        return uploader_cls(**upload_stager_kwargs)
+
+    @staticmethod
+    def get_upload_stager(dest: str, options: dict[str, Any]) -> Optional[UploadStagerT]:
+        if custom_stager := options.get("custom_stager"):
+            return BaseCmd.get_custom_stager(
+                stager_reference=custom_stager,
+                stager_config_kwargs=options.get("custom_stager_config_kwargs"),
+            )
+        dest_entry = destination_registry[dest]
+        upload_stager_kwargs: dict[str, Any] = {}
+        if upload_stager_config_cls := dest_entry.upload_stager_config:
+            upload_stager_kwargs["upload_stager_config"] = extract_config(
+                flat_data=options, config=upload_stager_config_cls
+            )
+        if upload_stager_cls := dest_entry.upload_stager:
+            return upload_stager_cls(**upload_stager_kwargs)
+        return None
+
+    @staticmethod
+    def get_uploader(dest, options: dict[str, Any]) -> UploaderT:
+        dest_entry = destination_registry[dest]
+        uploader_kwargs: dict[str, Any] = {}
+        if uploader_config_cls := dest_entry.uploader_config:
+            uploader_kwargs["upload_config"] = extract_config(
+                flat_data=options, config=uploader_config_cls
+            )
+        if connection_config_cls := dest_entry.connection_config:
+            uploader_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        uploader_cls = dest_entry.uploader
+        return uploader_cls(**uploader_kwargs)

unstructured_ingest/v2/cli/base/dest.py

@@ -0,0 +1,85 @@
+import logging
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.v2.cli.base.cmd import BaseCmd
+from unstructured_ingest.v2.cli.utils.click import Dict, conform_click_options
+from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+
+
+@dataclass
+class DestCmd(BaseCmd):
+    registry_entry: DestinationRegistryEntry
+
+    def get_registry_options(self):
+        options = []
+        configs = [
+            config
+            for config in [
+                self.registry_entry.uploader_config,
+                self.registry_entry.upload_stager_config,
+                self.registry_entry.connection_config,
+            ]
+            if config
+        ]
+        for config in configs:
+            options.extend(options_from_base_model(model=config))
+        options = self.consolidate_options(options=options)
+        return options
+
+    def cmd(self, ctx: click.Context, **options) -> None:
+        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
+        if not ctx.parent:
+            raise click.ClickException("destination command called without a parent")
+        if not ctx.parent.info_name:
+            raise click.ClickException("parent command missing info name")
+        source_cmd = ctx.parent.info_name.replace("-", "_")
+        source_options: dict = ctx.parent.params if ctx.parent else {}
+        conform_click_options(options)
+        try:
+            pipeline = self.get_pipeline(
+                src=source_cmd,
+                source_options=source_options,
+                dest=self.cmd_name,
+                destination_options=options,
+            )
+            pipeline.run()
+        except Exception as e:
+            logger.error(f"failed to run destination command {self.cmd_name}: {e}", exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_cmd(self) -> click.Command:
+        # Dynamically create the command without the use of click decorators
+        fn = self.cmd
+        fn = click.pass_context(fn)
+        cmd = click.command(fn)
+        if not isinstance(cmd, click.core.Command):
+            raise ValueError(f"generated command was not of expected type Command: {type(cmd)}")
+        cmd.name = self.cli_cmd_name
+        cmd.short_help = "v2"
+        cmd.invoke_without_command = True
+        self.add_options(cmd)
+        cmd.params.append(
+            click.Option(
+                ["--custom-stager"],
+                required=False,
+                type=str,
+                default=None,
+                help="Pass a pointer to a custom upload stager to use, "
+                "must be in format '<module>:<attribute>'",
+            )
+        )
+        cmd.params.append(
+            click.Option(
+                ["--custom-stager-config-kwargs"],
+                required=False,
+                type=Dict(),
+                default=None,
+                help="Any kwargs to instantiate the configuration "
+                "associated with the customer stager",
+            )
+        )
+        return cmd

unstructured_ingest/v2/cli/base/importer.py

@@ -0,0 +1,34 @@
+import importlib
+from typing import Any
+
+
+class ImportFromStringError(Exception):
+    pass
+
+
+def import_from_string(import_str: Any) -> Any:
+    if not isinstance(import_str, str):
+        return import_str
+
+    module_str, _, attrs_str = import_str.partition(":")
+    if not module_str or not attrs_str:
+        message = 'Import string "{import_str}" must be in format "<module>:<attribute>".'
+        raise ImportFromStringError(message.format(import_str=import_str))
+
+    try:
+        module = importlib.import_module(module_str)
+    except ModuleNotFoundError as exc:
+        if exc.name != module_str:
+            raise exc from None
+        message = 'Could not import module "{module_str}".'
+        raise ImportFromStringError(message.format(module_str=module_str))
+
+    instance = module
+    try:
+        for attr_str in attrs_str.split("."):
+            instance = getattr(instance, attr_str)
+    except AttributeError:
+        message = 'Attribute "{attrs_str}" not found in module "{module_str}".'
+        raise ImportFromStringError(message.format(attrs_str=attrs_str, module_str=module_str))
+
+    return instance