unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/v2/cli/configs/chunk.py

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+
+import click
+from unstructured.chunking import CHUNK_MAX_CHARS_DEFAULT, CHUNK_MULTI_PAGE_DEFAULT
+
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+
+
+@dataclass
+class ChunkerCliConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--chunking-strategy"],
+                type=str,
+                default=None,
+                help="The rule-set to use to form chunks. Omit to disable chunking.",
+            ),
+            click.Option(
+                ["--chunk-combine-text-under-n-chars"],
+                type=int,
+                help=(
+                    "Combine consecutive chunks when the first does not exceed this length and"
+                    " the second will fit without exceeding the hard-maximum length. Only"
+                    " operative for 'by_title' chunking-strategy."
+                ),
+            ),
+            click.Option(
+                ["--chunk-include-orig-elements/--chunk-no-include-orig-elements"],
+                is_flag=True,
+                default=True,
+                help=(
+                    "When chunking, add the original elements consolidated to form each chunk to"
+                    " `.metadata.orig_elements` on that chunk."
+                ),
+            ),
+            click.Option(
+                ["--chunk-max-characters"],
+                type=int,
+                default=CHUNK_MAX_CHARS_DEFAULT,
+                show_default=True,
+                help=(
+                    "Hard maximum chunk length. No chunk will exceed this length. An oversized"
+                    " element will be divided by text-splitting to fit this window."
+                ),
+            ),
+            click.Option(
+                ["--chunk-multipage-sections/--chunk-no-multipage-sections"],
+                is_flag=True,
+                default=CHUNK_MULTI_PAGE_DEFAULT,
+                help=(
+                    "Ignore page boundaries when chunking such that elements from two different"
+                    " pages can appear in the same chunk. Only operative for 'by_title'"
+                    " chunking-strategy."
+                ),
+            ),
+            click.Option(
+                ["--chunk-new-after-n-chars"],
+                type=int,
+                help=(
+                    "Soft-maximum chunk length. Another element will not be added to a chunk of"
+                    " this length even when it would fit without exceeding the hard-maximum"
+                    " length."
+                ),
+            ),
+            click.Option(
+                ["--chunk-overlap"],
+                type=int,
+                default=0,
+                show_default=True,
+                help=(
+                    "Prefix chunk text with last overlap=N characters of prior chunk. Only"
+                    " applies to oversized chunks divided by text-splitting. To apply overlap to"
+                    " non-oversized chunks use the --overlap-all option."
+                ),
+            ),
+            click.Option(
+                ["--chunk-overlap-all"],
+                is_flag=True,
+                default=False,
+                help=(
+                    "Apply overlap to chunks formed from whole elements as well as those formed"
+                    " by text-splitting oversized elements. Overlap length is take from --overlap"
+                    " option value."
+                ),
+            ),
+        ]
+        return options

unstructured_ingest/v2/cli/configs/embed.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass
+from typing import Any
+
+import click
+from dataclasses_json.core import Json
+from unstructured.embed import EMBEDDING_PROVIDER_TO_CLASS_MAP
+
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+
+
+@dataclass
+class EmbedderCliConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--embedding-provider"],
+                help="Type of the embedding class to be used. Can be one of: "
+                f"{list(EMBEDDING_PROVIDER_TO_CLASS_MAP)}",
+                type=click.Choice(list(EMBEDDING_PROVIDER_TO_CLASS_MAP)),
+            ),
+            click.Option(
+                ["--embedding-api-key"],
+                help="API key for the embedding model, for the case an API key is needed.",
+                type=str,
+                default=None,
+            ),
+            click.Option(
+                ["--embedding-model-name"],
+                help="Embedding model name, if needed. "
+                "Chooses a particular LLM between different options, to embed with it.",
+                type=str,
+                default=None,
+            ),
+            click.Option(
+                ["--embedding-aws-access-key-id"],
+                help="AWS access key used for AWS-based embedders, such as bedrock",
+                type=str,
+                default=None,
+            ),
+            click.Option(
+                ["--embedding-aws-secret-access-key"],
+                help="AWS secret key used for AWS-based embedders, such as bedrock",
+                type=str,
+                default=None,
+            ),
+            click.Option(
+                ["--embedding-aws-region"],
+                help="AWS region used for AWS-based embedders, such as bedrock",
+                type=str,
+                default="us-west-2",
+            ),
+        ]
+        return options
+
+    @classmethod
+    def from_dict(cls, kvs: Json, **kwargs: Any):
+        """
+        Extension of the dataclass from_dict() to avoid a naming conflict with other CLI params.
+        This allows CLI arguments to be prepended with embedding_ during CLI invocation but
+        doesn't require that as part of the field names in this class
+        """
+        if isinstance(kvs, dict):
+            new_kvs = {
+                k[len("embedding_") :]: v  # noqa: E203
+                for k, v in kvs.items()
+                if k.startswith("embedding_")
+            }
+            if len(new_kvs.keys()) == 0:
+                return None
+            if not new_kvs.get("provider"):
+                return None
+            return super().from_dict(new_kvs, **kwargs)
+        return super().from_dict(kvs, **kwargs)

unstructured_ingest/v2/cli/configs/partition.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.cli.utils import DelimitedString, Dict
+
+
+@dataclass
+class PartitionerCliConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--strategy"],
+                default="auto",
+                help="The method that will be used to process the documents. "
+                "Default: auto. Other strategies include `fast` and `hi_res`.",
+            ),
+            click.Option(
+                ["--ocr-languages"],
+                default=None,
+                type=DelimitedString(delimiter="+"),
+                help="A list of language packs to specify which languages to use for OCR, "
+                "separated by '+' e.g. 'eng+deu' to use the English and German language packs. "
+                "The appropriate Tesseract "
+                "language pack needs to be installed.",
+            ),
+            click.Option(
+                ["--encoding"],
+                default=None,
+                help="Text encoding to use when reading documents. By default the encoding is "
+                "detected automatically.",
+            ),
+            click.Option(
+                ["--skip-infer-table-types"],
+                type=DelimitedString(),
+                default=None,
+                help="Optional list of document types to skip table extraction on",
+            ),
+            click.Option(
+                ["--additional-partition-args"],
+                type=Dict(),
+                help="A json string representation of values to pass through to partition()",
+            ),
+            click.Option(
+                ["--fields-include"],
+                type=DelimitedString(),
+                default=["element_id", "text", "type", "metadata", "embeddings"],
+                help="Comma-delimited list. If set, include the specified top-level "
+                "fields in an element.",
+            ),
+            click.Option(
+                ["--flatten-metadata"],
+                is_flag=True,
+                default=False,
+                help="Results in flattened json elements. "
+                "Specifically, the metadata key values are brought to "
+                "the top-level of the element, and the `metadata` key itself is removed.",
+            ),
+            click.Option(
+                ["--metadata-include"],
+                default=[],
+                type=DelimitedString(),
+                help="Comma-delimited list. If set, include the specified metadata "
+                "fields if they exist and drop all other fields. ",
+            ),
+            click.Option(
+                ["--metadata-exclude"],
+                default=[],
+                type=DelimitedString(),
+                help="Comma-delimited list. If set, drop the specified metadata "
+                "fields if they exist.",
+            ),
+            click.Option(
+                ["--partition-by-api"],
+                is_flag=True,
+                default=False,
+                help="Use a remote API to partition the files."
+                " Otherwise, use the function from partition.auto",
+            ),
+            click.Option(
+                ["--partition-endpoint"],
+                default="https://api.unstructured.io/general/v0/general",
+                help="If partitioning via api, use the following host. "
+                "Default: https://api.unstructured.io/general/v0/general",
+            ),
+            click.Option(
+                ["--api-key"],
+                default=None,
+                help="API Key for partition endpoint.",
+            ),
+            click.Option(
+                ["--hi-res-model-name"],
+                default=None,
+                help="Model name for hi-res strategy.",
+            ),
+        ]
+        return options

unstructured_ingest/v2/cli/configs/processor.py

@@ -0,0 +1,88 @@
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.interfaces.processor import DEFAULT_WORK_DIR
+
+
+@dataclass
+class ProcessorCliConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--reprocess"],
+                is_flag=True,
+                default=False,
+                help="Reprocess a downloaded file even if the relevant structured "
+                "output .json file in output directory already exists.",
+            ),
+            click.Option(
+                ["--work-dir"],
+                type=str,
+                default=DEFAULT_WORK_DIR,
+                show_default=True,
+                help="Where to place working files when processing each step",
+            ),
+            click.Option(
+                ["--num-processes"],
+                default=2,
+                show_default=True,
+                type=click.IntRange(min=1),
+                help="Number of parallel processes with which to process docs",
+            ),
+            click.Option(
+                ["--max-connections"],
+                default=None,
+                show_default=True,
+                type=click.IntRange(min=1),
+                help="Max number of connections allowed when running an async step",
+            ),
+            click.Option(
+                ["--raise-on-error"],
+                is_flag=True,
+                default=False,
+                help="Is set, will raise error if any doc in the pipeline fail. Otherwise will "
+                "log error and continue with other docs",
+            ),
+            click.Option(
+                ["--re-download"],
+                is_flag=True,
+                default=False,
+                help="Re-download files even if they are already present in download dir.",
+            ),
+            click.Option(
+                ["--preserve-downloads"],
+                is_flag=True,
+                default=False,
+                help="Preserve downloaded files. Otherwise each file is removed "
+                "after being processed successfully.",
+            ),
+            click.Option(
+                ["--download-only"],
+                is_flag=True,
+                default=False,
+                help="Download any files that are not already present in either --download-dir or "
+                "the default download ~/.cache/... location in case --download-dir "
+                "is not specified and "
+                "skip processing them through unstructured.",
+            ),
+            click.Option(
+                ["--max-docs"],
+                default=None,
+                type=int,
+                help="If specified, process at most the specified number of documents.",
+            ),
+            click.Option(
+                ["--uncompress"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="Uncompress any archived files. Currently supporting zip and tar "
+                "files based on file extension.",
+            ),
+            click.Option(["--verbose"], is_flag=True, default=False),
+            click.Option(["--tqdm"], is_flag=True, default=False, help="Show progress bar"),
+        ]
+        return options

unstructured_ingest/v2/cli/interfaces.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+
+import click
+
+
+class CliConfig(ABC):
+    @staticmethod
+    @abstractmethod
+    def get_cli_options() -> list[click.Option]:
+        pass
+
+    @classmethod
+    def add_cli_options(cls, cmd: click.Command) -> None:
+        options_to_add = cls.get_cli_options()
+        CliConfig.add_params(cmd, params=options_to_add)
+
+    @staticmethod
+    def add_params(cmd: click.Command, params: list[click.Parameter]):
+        existing_opts = []
+        for param in cmd.params:
+            existing_opts.extend(param.opts)
+        for param in params:
+            for opt in param.opts:
+                if opt in existing_opts:
+                    raise ValueError(f"{opt} is already defined on the command {cmd.name}")
+                existing_opts.append(opt)
+                cmd.params.append(param)

unstructured_ingest/v2/cli/utils.py

@@ -0,0 +1,240 @@
+import json
+import os.path
+import sys
+from dataclasses import fields, is_dataclass
+from gettext import gettext, ngettext
+from gettext import gettext as _
+from pathlib import Path
+from typing import Any, ForwardRef, Optional, Type, TypeVar, Union, get_args, get_origin
+
+import click
+
+from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from unstructured_ingest.v2.logger import logger
+
+
+def conform_click_options(options: dict):
+    # Click sets all multiple fields as tuple, this needs to be updated to list
+    for k, v in options.items():
+        if isinstance(v, tuple):
+            options[k] = list(v)
+
+
+class Dict(click.ParamType):
+    name = "dict"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return json.loads(value)
+        except json.JSONDecodeError:
+            self.fail(
+                gettext(
+                    "{value} is not a valid json value.",
+                ).format(value=value),
+                param,
+                ctx,
+            )
+
+
+class FileOrJson(click.ParamType):
+    name = "file-or-json"
+
+    def __init__(self, allow_raw_str: bool = False):
+        self.allow_raw_str = allow_raw_str
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        # check if valid file
+        full_path = os.path.abspath(os.path.expanduser(value))
+        if os.path.isfile(full_path):
+            return str(Path(full_path).resolve())
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except json.JSONDecodeError:
+                if self.allow_raw_str:
+                    return value
+        self.fail(
+            gettext(
+                "{value} is not a valid json string nor an existing filepath.",
+            ).format(value=value),
+            param,
+            ctx,
+        )
+
+
+class DelimitedString(click.ParamType):
+    name = "delimited-string"
+
+    def __init__(self, delimiter: str = ",", choices: Optional[list[str]] = None):
+        self.choices = choices if choices else []
+        self.delimiter = delimiter
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        # In case a list is provided as the default, will not break
+        if isinstance(value, list):
+            split = [str(v).strip() for v in value]
+        else:
+            split = [v.strip() for v in value.split(self.delimiter)]
+        if not self.choices:
+            return split
+        choices_str = ", ".join(map(repr, self.choices))
+        for s in split:
+            if s not in self.choices:
+                self.fail(
+                    ngettext(
+                        "{value!r} is not {choice}.",
+                        "{value!r} is not one of {choices}.",
+                        len(self.choices),
+                    ).format(value=s, choice=choices_str, choices=choices_str),
+                    param,
+                    ctx,
+                )
+        return split
+
+
+EnhancedDataClassJsonMixinT = TypeVar(
+    "EnhancedDataClassJsonMixinT", bound=EnhancedDataClassJsonMixin
+)
+
+
+def extract_config(
+    flat_data: dict, config: Type[EnhancedDataClassJsonMixinT]
+) -> EnhancedDataClassJsonMixinT:
+    """
+    To be able to extract a nested dataclass from a flat dictionary (as in one coming
+    from a click-based options input), the config class is dynamically looked through for
+    nested dataclass fields and new nested dictionaries are created to conform to the
+    shape the overall class expects when parsing from a dict. During the process, this will create
+    copies of the original dictionary to avoid pruning fields but this isn't a
+    problem since the `from_dict()` method ignores unneeded values.
+
+    Not handling more complex edge cases for now such as nested types i.e Union[List[List[...]]]
+    """
+
+    def conform_dict(inner_d: dict, inner_config: Type[EnhancedDataClassJsonMixinT]):
+        # Catch edge cases (i.e. Dict[str, ...]) where underlying type is not a concrete Class,
+        # causing 'issubclass() arg 1 must be a class' errors, return False
+        def is_subclass(instance, class_type) -> bool:
+            try:
+                return issubclass(instance, class_type)
+            except Exception:
+                return False
+
+        dd = inner_d.copy()
+        for field in fields(inner_config):
+            f_type = field.type
+            # typing can be defined using a string, in which case it needs to be resolved
+            # to the actual type. following logic is cherry picked from the typing
+            # get_type_hints() since type resolution can be expensive, only do it
+            # when the type is a string
+            if isinstance(f_type, str):
+                try:
+                    base_globals = sys.modules[inner_config.__module__].__dict__
+                    for_ref = ForwardRef(f_type, is_argument=False, is_class=True)
+                    f_type = for_ref._evaluate(
+                        globalns=base_globals, localns=None, recursive_guard=frozenset()
+                    )
+                except NameError as e:
+                    logger.warning(f"couldn't resolve type {f_type}: {e}")
+            # Handle the case where the type of a value if a Union (possibly optional)
+            if get_origin(f_type) is Union:
+                union_values = get_args(f_type)
+                # handle List types
+                union_values = [
+                    get_args(u)[0] if get_origin(u) is list else u for u in union_values
+                ]
+                # Ignore injected NoneType when optional
+                concrete_union_values = [v for v in union_values if not is_subclass(v, type(None))]
+                dataclass_union_values = [v for v in concrete_union_values if is_dataclass(v)]
+                non_dataclass_union_values = [
+                    v for v in concrete_union_values if not is_dataclass(v)
+                ]
+                if not dataclass_union_values:
+                    continue
+                # Check if the key for this field already exists in the dictionary,
+                # if so it might map to one of these non dataclass fields and this
+                # can't be enforced
+                if non_dataclass_union_values and field.name in dd:
+                    continue
+                if len(dataclass_union_values) > 1:
+                    logger.warning(
+                        "more than one dataclass type possible for field {}, "
+                        "not extracting: {}".format(field.name, ", ".join(dataclass_union_values))
+                    )
+                    continue
+                f_type = dataclass_union_values[0]
+            origin = get_origin(f_type)
+            if origin:
+                f_type = origin
+            if is_subclass(f_type, EnhancedDataClassJsonMixin):
+                dd[field.name] = conform_dict(inner_d=dd, inner_config=f_type)
+        return dd
+
+    adjusted_dict = conform_dict(inner_d=flat_data, inner_config=config)
+    return config.from_dict(adjusted_dict, apply_name_overload=False)
+
+
+class Group(click.Group):
+    def parse_args(self, ctx, args):
+        """
+        This allows for subcommands to be called with the --help flag without breaking
+        if parent command is missing any of its required parameters
+        """
+
+        try:
+            return super().parse_args(ctx, args)
+        except click.MissingParameter:
+            if "--help" not in args:
+                raise
+
+            # remove the required params so that help can display
+            for param in self.params:
+                param.required = False
+            return super().parse_args(ctx, args)
+
+    def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
+        """
+        Copy of the original click.Group format_commands() method but replacing
+        'Commands' -> 'Destinations'
+        """
+        commands = []
+        for subcommand in self.list_commands(ctx):
+            cmd = self.get_command(ctx, subcommand)
+            # What is this, the tool lied about a command.  Ignore it
+            if cmd is None:
+                continue
+            if cmd.hidden:
+                continue
+
+            commands.append((subcommand, cmd))
+
+        # allow for 3 times the default spacing
+        if len(commands):
+            if formatter.width:
+                limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands)
+            else:
+                limit = -6 - max(len(cmd[0]) for cmd in commands)
+
+            rows = []
+            for subcommand, cmd in commands:
+                help = cmd.get_short_help_str(limit)
+                rows.append((subcommand, help))
+
+            if rows:
+                with formatter.section(_("Destinations")):
+                    formatter.write_dl(rows)

unstructured_ingest/v2/example.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.pipeline.pipeline import Pipeline
+from unstructured_ingest.v2.processes.chunker import ChunkerConfig
+from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
+    S3ConnectionConfig,
+    S3DownloaderConfig,
+    S3IndexerConfig,
+)
+from unstructured_ingest.v2.processes.connectors.local import (
+    LocalUploaderConfig,
+)
+from unstructured_ingest.v2.processes.embedder import EmbedderConfig
+from unstructured_ingest.v2.processes.partitioner import PartitionerConfig
+
+base_path = Path(__file__).parent.parent.parent.parent
+docs_path = base_path / "example-docs"
+work_dir = base_path / "tmp_ingest"
+output_path = work_dir / "output"
+download_path = work_dir / "download"
+
+if __name__ == "__main__":
+    logger.info(f"Writing all content in: {work_dir.resolve()}")
+    Pipeline.from_configs(
+        context=ProcessorConfig(
+            work_dir=str(work_dir.resolve()), tqdm=True, reprocess=True, verbose=True
+        ),
+        indexer_config=S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/"),
+        downloader_config=S3DownloaderConfig(download_dir=download_path),
+        source_connection_config=S3ConnectionConfig(anonymous=True),
+        partitioner_config=PartitionerConfig(strategy="fast"),
+        chunker_config=ChunkerConfig(chunking_strategy="by_title"),
+        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
+    ).run()

unstructured_ingest/v2/interfaces/__init__.py

@@ -0,0 +1,29 @@
+from .connector import AccessConfig, BaseConnector, ConnectionConfig
+from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
+from .file_data import FileData, SourceIdentifiers
+from .indexer import Indexer, IndexerConfig
+from .process import BaseProcess
+from .processor import ProcessorConfig
+from .upload_stager import UploadStager, UploadStagerConfig
+from .uploader import UploadContent, Uploader, UploaderConfig
+
+__all__ = [
+    "DownloadResponse",
+    "download_responses",
+    "Downloader",
+    "DownloaderConfig",
+    "FileData",
+    "Indexer",
+    "IndexerConfig",
+    "BaseProcess",
+    "ProcessorConfig",
+    "UploadStager",
+    "UploadStagerConfig",
+    "Uploader",
+    "UploaderConfig",
+    "SourceIdentifiers",
+    "UploadContent",
+    "AccessConfig",
+    "ConnectionConfig",
+    "BaseConnector",
+]