unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.32"  # pragma: no cover

unstructured_ingest/cli/README.md

@@ -0,0 +1,28 @@
+# Ingest CLI
+This package helps map user input via a cli to the underlying ingest code to run a small ETL pipeline.
+
+## Design Reference
+[cli.py](cli.py) is the main entrypoint to run the cli itself. The key points for this is the interaction between all
+source and destination connectors.
+
+To manually run the cli:
+```shell
+PYTHONPATH=. python unstructured_ingest/main.py --help
+```
+
+The `main.py` file simply wraps the generated Click command created in `cli.py`.
+
+### Source Commands
+All source commands are added as sub commands to the parent ingest Click group. This allows each command to map to
+different connectors with shared and unique parameters.
+
+### Destination Commands
+All destination commands are added as sub commands to each parent source command. This allows each invocation of the source
+sub command to display all possible destination subcommands. The code un [utils.py](./utils.py) helps structure the
+generated text from the Click library to be more intuitive on this approach (i.e. list sub commands as  `Destinations`).
+
+### Configs
+The configs in [configs/](./configs) and connector specific ones in [cmds/](./cmds) help surface all user parameters that
+are needed to marshall the input dictionary from Click into all the respective configs needed to create a full pipeline run.
+Because click returns a flat dictionary of user inputs, the `extract_config` method in `utils.py` helps deserialize this dictionary
+into dataclasses that have nested fields (such as access configs).

unstructured_ingest/cli/base/__init__.py

@@ -0,0 +1,4 @@
+from .dest import DestCmd
+from .src import SrcCmd
+
+__all__ = ["SrcCmd", "DestCmd"]

unstructured_ingest/cli/base/cmd.py

@@ -0,0 +1,269 @@
+import inspect
+from abc import ABC, abstractmethod
+from collections import Counter
+from dataclasses import dataclass, field, fields
+from typing import Any, Optional, Type, TypeVar
+
+import click
+from pydantic import BaseModel
+
+from unstructured_ingest.cli.base.importer import import_from_string
+from unstructured_ingest.cli.utils.click import extract_config
+from unstructured_ingest.cli.utils.model_conversion import options_from_base_model, post_check
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.pipeline.pipeline import Pipeline
+from unstructured_ingest.processes.chunker import Chunker, ChunkerConfig
+from unstructured_ingest.processes.connector_registry import (
+    DownloaderT,
+    IndexerT,
+    RegistryEntry,
+    UploaderT,
+    UploadStager,
+    UploadStagerConfig,
+    UploadStagerT,
+    destination_registry,
+    source_registry,
+)
+from unstructured_ingest.processes.connectors.local import LocalUploader, LocalUploaderConfig
+from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.processes.filter import Filterer, FiltererConfig
+from unstructured_ingest.processes.partitioner import Partitioner, PartitionerConfig
+
+CommandT = TypeVar("CommandT", bound=click.Command)
+
+
+@dataclass
+class BaseCmd(ABC):
+    cmd_name: str
+    registry_entry: RegistryEntry
+    default_configs: list[Type[BaseModel]] = field(default_factory=list)
+
+    @abstractmethod
+    def get_registry_options(self):
+        pass
+
+    def get_default_options(self) -> list[click.Option]:
+        options = []
+        for extra in self.default_configs:
+            options.extend(options_from_base_model(model=extra))
+        return options
+
+    @classmethod
+    def consolidate_options(cls, options: list[click.Option]) -> list[click.Option]:
+        option_names = [option.name for option in options]
+        duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+        if not duplicate_names:
+            return options
+        consolidated_options = []
+        current_names = []
+        for option in options:
+            if option.name not in current_names:
+                current_names.append(option.name)
+                consolidated_options.append(option)
+                continue
+            existing_option = next(o for o in consolidated_options if o.name == option.name)
+            if existing_option.__dict__ == option.__dict__:
+                continue
+            option_diff = cls.get_options_diff(o1=option, o2=existing_option)
+            raise ValueError(
+                "Conflicting duplicate {} option defined: {}".format(
+                    option.name, " | ".join([f"{d[0]}: {d[1]}" for d in option_diff])
+                )
+            )
+        return consolidated_options
+
+    @staticmethod
+    def get_options_diff(o1: click.Option, o2: click.Option):
+        o1_dict = o1.__dict__
+        o2_dict = o2.__dict__
+        for d in [o1_dict, o2_dict]:
+            d["opts"] = ",".join(d["opts"])
+            d["secondary_opts"] = ",".join(d["secondary_opts"])
+        option_diff = set(o1_dict.items()) ^ set(o2_dict.items())
+        return option_diff
+
+    @property
+    def cmd_name_key(self):
+        return self.cmd_name.replace("-", "_")
+
+    @property
+    def cli_cmd_name(self):
+        return self.cmd_name.replace("_", "-")
+
+    @abstractmethod
+    def cmd(self, ctx: click.Context, **options) -> None:
+        pass
+
+    def add_options(self, cmd: CommandT) -> CommandT:
+        options = self.get_registry_options()
+        options.extend(self.get_default_options())
+        post_check(options=options, name=cmd.name)
+        cmd.params.extend(options)
+        return cmd
+
+    def get_pipeline(
+        self,
+        src: str,
+        source_options: dict[str, Any],
+        dest: Optional[str] = None,
+        destination_options: Optional[dict[str, Any]] = None,
+    ) -> Pipeline:
+        logger.debug(
+            f"creating pipeline from cli using source {src} with options: {source_options}"
+        )
+        pipeline_kwargs: dict[str, Any] = {
+            "context": self.get_processor_config(options=source_options),
+            "downloader": self.get_downloader(src=src, options=source_options),
+            "indexer": self.get_indexer(src=src, options=source_options),
+            "partitioner": self.get_partitioner(options=source_options),
+        }
+        if chunker := self.get_chunker(options=source_options):
+            pipeline_kwargs["chunker"] = chunker
+        if filterer := self.get_filterer(options=source_options):
+            pipeline_kwargs["filterer"] = filterer
+        if embedder := self.get_embedder(options=source_options):
+            pipeline_kwargs["embedder"] = embedder
+        if dest:
+            logger.debug(
+                f"setting destination on pipeline {dest} with options: {destination_options}"
+            )
+            if uploader_stager := self.get_upload_stager(dest=dest, options=destination_options):
+                pipeline_kwargs["stager"] = uploader_stager
+            pipeline_kwargs["uploader"] = self.get_uploader(dest=dest, options=destination_options)
+        else:
+            # Default to local uploader
+            # TODO remove after v1 no longer supported
+            destination_options = destination_options or {}
+            if "output_dir" not in destination_options:
+                destination_options["output_dir"] = source_options["output_dir"]
+            pipeline_kwargs["uploader"] = self.get_default_uploader(options=destination_options)
+        return Pipeline(**pipeline_kwargs)
+
+    @staticmethod
+    def get_default_uploader(options: dict[str, Any]) -> UploaderT:
+        uploader_config = extract_config(flat_data=options, config=LocalUploaderConfig)
+        return LocalUploader(upload_config=uploader_config)
+
+    @staticmethod
+    def get_chunker(options: dict[str, Any]) -> Optional[Chunker]:
+        chunker_config = extract_config(flat_data=options, config=ChunkerConfig)
+        if not chunker_config.chunking_strategy:
+            return None
+        return Chunker(config=chunker_config)
+
+    @staticmethod
+    def get_filterer(options: dict[str, Any]) -> Optional[Filterer]:
+        filterer_configs = extract_config(flat_data=options, config=FiltererConfig)
+        if not filterer_configs.model_dump():
+            return None
+        return Filterer(config=filterer_configs)
+
+    @staticmethod
+    def get_embedder(options: dict[str, Any]) -> Optional[Embedder]:
+        embedder_config = extract_config(flat_data=options, config=EmbedderConfig)
+        if not embedder_config.embedding_provider:
+            return None
+        return Embedder(config=embedder_config)
+
+    @staticmethod
+    def get_partitioner(options: dict[str, Any]) -> Partitioner:
+        partitioner_config = extract_config(flat_data=options, config=PartitionerConfig)
+        return Partitioner(config=partitioner_config)
+
+    @staticmethod
+    def get_processor_config(options: dict[str, Any]) -> ProcessorConfig:
+        return extract_config(flat_data=options, config=ProcessorConfig)
+
+    @staticmethod
+    def get_indexer(src: str, options: dict[str, Any]) -> IndexerT:
+        source_entry = source_registry[src]
+        indexer_kwargs: dict[str, Any] = {}
+        if indexer_config_cls := source_entry.indexer_config:
+            indexer_kwargs["index_config"] = extract_config(
+                flat_data=options, config=indexer_config_cls
+            )
+        if connection_config_cls := source_entry.connection_config:
+            indexer_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        indexer_cls = source_entry.indexer
+        return indexer_cls(**indexer_kwargs)
+
+    @staticmethod
+    def get_downloader(src: str, options: dict[str, Any]) -> DownloaderT:
+        source_entry = source_registry[src]
+        downloader_kwargs: dict[str, Any] = {}
+        if downloader_config_cls := source_entry.downloader_config:
+            downloader_kwargs["download_config"] = extract_config(
+                flat_data=options, config=downloader_config_cls
+            )
+        if connection_config_cls := source_entry.connection_config:
+            downloader_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        downloader_cls = source_entry.downloader
+        return downloader_cls(**downloader_kwargs)
+
+    @staticmethod
+    def get_custom_stager(
+        stager_reference: str, stager_config_kwargs: Optional[dict] = None
+    ) -> Optional[UploadStagerT]:
+        uploader_cls = import_from_string(stager_reference)
+        if not inspect.isclass(uploader_cls):
+            raise ValueError(
+                f"custom stager must be a reference to a python class, got: {type(uploader_cls)}"
+            )
+        if not issubclass(uploader_cls, UploadStager):
+            raise ValueError(
+                "custom stager must be an implementation of the UploadStager interface"
+            )
+        fields_dict = {f.name: f.type for f in fields(uploader_cls)}
+        upload_stager_config_cls = fields_dict["upload_stager_config"]
+        if not inspect.isclass(upload_stager_config_cls):
+            raise ValueError(
+                f"custom stager config must be a class, got: {type(upload_stager_config_cls)}"
+            )
+        if not issubclass(upload_stager_config_cls, UploadStagerConfig):
+            raise ValueError(
+                "custom stager config must be an implementation "
+                "of the UploadStagerUploadStagerConfig interface"
+            )
+        upload_stager_kwargs: dict[str, Any] = {}
+        if stager_config_kwargs:
+            upload_stager_kwargs["upload_stager_config"] = upload_stager_config_cls(
+                **stager_config_kwargs
+            )
+        return uploader_cls(**upload_stager_kwargs)
+
+    @staticmethod
+    def get_upload_stager(dest: str, options: dict[str, Any]) -> Optional[UploadStagerT]:
+        if custom_stager := options.get("custom_stager"):
+            return BaseCmd.get_custom_stager(
+                stager_reference=custom_stager,
+                stager_config_kwargs=options.get("custom_stager_config_kwargs"),
+            )
+        dest_entry = destination_registry[dest]
+        upload_stager_kwargs: dict[str, Any] = {}
+        if upload_stager_config_cls := dest_entry.upload_stager_config:
+            upload_stager_kwargs["upload_stager_config"] = extract_config(
+                flat_data=options, config=upload_stager_config_cls
+            )
+        if upload_stager_cls := dest_entry.upload_stager:
+            return upload_stager_cls(**upload_stager_kwargs)
+        return None
+
+    @staticmethod
+    def get_uploader(dest, options: dict[str, Any]) -> UploaderT:
+        dest_entry = destination_registry[dest]
+        uploader_kwargs: dict[str, Any] = {}
+        if uploader_config_cls := dest_entry.uploader_config:
+            uploader_kwargs["upload_config"] = extract_config(
+                flat_data=options, config=uploader_config_cls
+            )
+        if connection_config_cls := dest_entry.connection_config:
+            uploader_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        uploader_cls = dest_entry.uploader
+        return uploader_cls(**uploader_kwargs)

unstructured_ingest/cli/base/dest.py

@@ -0,0 +1,84 @@
+import logging
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.cmd import BaseCmd
+from unstructured_ingest.cli.utils.click import Dict, conform_click_options
+from unstructured_ingest.cli.utils.model_conversion import options_from_base_model
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+
+
+@dataclass
+class DestCmd(BaseCmd):
+    registry_entry: DestinationRegistryEntry
+
+    def get_registry_options(self):
+        options = []
+        configs = [
+            config
+            for config in [
+                self.registry_entry.uploader_config,
+                self.registry_entry.upload_stager_config,
+                self.registry_entry.connection_config,
+            ]
+            if config
+        ]
+        for config in configs:
+            options.extend(options_from_base_model(model=config))
+        options = self.consolidate_options(options=options)
+        return options
+
+    def cmd(self, ctx: click.Context, **options) -> None:
+        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
+        if not ctx.parent:
+            raise click.ClickException("destination command called without a parent")
+        if not ctx.parent.info_name:
+            raise click.ClickException("parent command missing info name")
+        source_cmd = ctx.parent.info_name.replace("-", "_")
+        source_options: dict = ctx.parent.params if ctx.parent else {}
+        conform_click_options(options)
+        try:
+            pipeline = self.get_pipeline(
+                src=source_cmd,
+                source_options=source_options,
+                dest=self.cmd_name,
+                destination_options=options,
+            )
+            pipeline.run()
+        except Exception as e:
+            logger.error(f"failed to run destination command {self.cmd_name}: {e}", exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_cmd(self) -> click.Command:
+        # Dynamically create the command without the use of click decorators
+        fn = self.cmd
+        fn = click.pass_context(fn)
+        cmd = click.command(fn)
+        if not isinstance(cmd, click.core.Command):
+            raise ValueError(f"generated command was not of expected type Command: {type(cmd)}")
+        cmd.name = self.cli_cmd_name
+        cmd.invoke_without_command = True
+        self.add_options(cmd)
+        cmd.params.append(
+            click.Option(
+                ["--custom-stager"],
+                required=False,
+                type=str,
+                default=None,
+                help="Pass a pointer to a custom upload stager to use, "
+                "must be in format '<module>:<attribute>'",
+            )
+        )
+        cmd.params.append(
+            click.Option(
+                ["--custom-stager-config-kwargs"],
+                required=False,
+                type=Dict(),
+                default=None,
+                help="Any kwargs to instantiate the configuration "
+                "associated with the customer stager",
+            )
+        )
+        return cmd

unstructured_ingest/cli/base/importer.py

@@ -0,0 +1,34 @@
+import importlib
+from typing import Any
+
+
+class ImportFromStringError(Exception):
+    pass
+
+
+def import_from_string(import_str: Any) -> Any:
+    if not isinstance(import_str, str):
+        return import_str
+
+    module_str, _, attrs_str = import_str.partition(":")
+    if not module_str or not attrs_str:
+        message = 'Import string "{import_str}" must be in format "<module>:<attribute>".'
+        raise ImportFromStringError(message.format(import_str=import_str))
+
+    try:
+        module = importlib.import_module(module_str)
+    except ModuleNotFoundError as exc:
+        if exc.name != module_str:
+            raise exc from None
+        message = 'Could not import module "{module_str}".'
+        raise ImportFromStringError(message.format(module_str=module_str))
+
+    instance = module
+    try:
+        for attr_str in attrs_str.split("."):
+            instance = getattr(instance, attr_str)
+    except AttributeError:
+        message = 'Attribute "{attrs_str}" not found in module "{module_str}".'
+        raise ImportFromStringError(message.format(attrs_str=attrs_str, module_str=module_str))
+
+    return instance

unstructured_ingest/cli/base/src.py

@@ -0,0 +1,75 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+import click
+from pydantic import BaseModel
+
+from unstructured_ingest.cli.base.cmd import BaseCmd
+from unstructured_ingest.cli.utils.click import Group, conform_click_options
+from unstructured_ingest.cli.utils.model_conversion import options_from_base_model
+from unstructured_ingest.interfaces import ProcessorConfig
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes import (
+    ChunkerConfig,
+    EmbedderConfig,
+    FiltererConfig,
+    PartitionerConfig,
+)
+from unstructured_ingest.processes.connector_registry import SourceRegistryEntry
+
+
+@dataclass
+class SrcCmd(BaseCmd):
+    registry_entry: SourceRegistryEntry
+    default_configs: list[BaseModel] = field(
+        default_factory=lambda: [
+            ProcessorConfig,
+            PartitionerConfig,
+            EmbedderConfig,
+            FiltererConfig,
+            ChunkerConfig,
+        ]
+    )
+
+    def get_registry_options(self):
+        options = []
+        configs = [
+            config
+            for config in [
+                self.registry_entry.connection_config,
+                self.registry_entry.indexer_config,
+                self.registry_entry.downloader_config,
+            ]
+            if config
+        ]
+        for config in configs:
+            options.extend(options_from_base_model(model=config))
+        options = self.consolidate_options(options=options)
+        return options
+
+    def cmd(self, ctx: click.Context, **options: dict[str, Any]) -> None:
+        if ctx.invoked_subcommand:
+            return
+
+        conform_click_options(options)
+        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
+        try:
+            pipeline = self.get_pipeline(src=self.cmd_name, source_options=options)
+            pipeline.run()
+        except Exception as e:
+            logger.error(f"failed to run source command {self.cmd_name}: {e}", exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_cmd(self) -> click.Group:
+        # Dynamically create the command without the use of click decorators
+        fn = self.cmd
+        fn = click.pass_context(fn)
+        cmd = click.group(fn, cls=Group)
+        if not isinstance(cmd, click.core.Group):
+            raise ValueError(f"generated src command was not of expected type Group: {type(cmd)}")
+        cmd.name = self.cli_cmd_name
+        cmd.invoke_without_command = True
+        self.add_options(cmd)
+
+        return cmd

unstructured_ingest/cli/cli.py

@@ -0,0 +1,24 @@
+import click
+
+from unstructured_ingest.cli.cmds import dest, src
+
+
+@click.group()
+def ingest():
+    pass
+
+
+def get_cmd() -> click.Command:
+    """Construct and return a Click command object representing the main command for the CLI.
+
+    This function adds all dest_subcommand(s) to each src_subcommand, and adds all of those
+    to the main command as nested subcommands.
+    """
+    cmd = ingest
+    # Add all subcommands
+    for src_subcommand in src:
+        # Add all destination subcommands
+        for dest_subcommand in dest:
+            src_subcommand.add_command(dest_subcommand)
+        cmd.add_command(src_subcommand)
+    return cmd

unstructured_ingest/cli/cmds.py

@@ -0,0 +1,14 @@
+import click
+
+from unstructured_ingest.cli.base import DestCmd, SrcCmd
+from unstructured_ingest.processes.connector_registry import (
+    destination_registry,
+    source_registry,
+)
+
+src_cmds = [SrcCmd(cmd_name=k, registry_entry=v) for k, v in source_registry.items()]
+dest_cmds = [DestCmd(cmd_name=k, registry_entry=v) for k, v in destination_registry.items()]
+
+src: list[click.Group] = [v.get_cmd() for v in src_cmds]
+
+dest: list[click.Command] = [v.get_cmd() for v in dest_cmds]