unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/v2/cli/base/src.py

@@ -0,0 +1,85 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+import click
+from pydantic import BaseModel
+
+from unstructured_ingest.v2.cli.base.cmd import BaseCmd
+from unstructured_ingest.v2.cli.utils.click import Group, conform_click_options
+from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes import (
+    ChunkerConfig,
+    EmbedderConfig,
+    FiltererConfig,
+    PartitionerConfig,
+)
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+
+@dataclass
+class SrcCmd(BaseCmd):
+    registry_entry: SourceRegistryEntry
+    default_configs: list[BaseModel] = field(
+        default_factory=lambda: [
+            ProcessorConfig,
+            PartitionerConfig,
+            EmbedderConfig,
+            FiltererConfig,
+            ChunkerConfig,
+        ]
+    )
+
+    def get_registry_options(self):
+        options = []
+        configs = [
+            config
+            for config in [
+                self.registry_entry.connection_config,
+                self.registry_entry.indexer_config,
+                self.registry_entry.downloader_config,
+            ]
+            if config
+        ]
+        for config in configs:
+            options.extend(options_from_base_model(model=config))
+        options = self.consolidate_options(options=options)
+        return options
+
+    def cmd(self, ctx: click.Context, **options: dict[str, Any]) -> None:
+        if ctx.invoked_subcommand:
+            return
+
+        conform_click_options(options)
+        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
+        try:
+            pipeline = self.get_pipeline(src=self.cmd_name, source_options=options)
+            pipeline.run()
+        except Exception as e:
+            logger.error(f"failed to run source command {self.cmd_name}: {e}", exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_cmd(self) -> click.Group:
+        # Dynamically create the command without the use of click decorators
+        fn = self.cmd
+        fn = click.pass_context(fn)
+        cmd = click.group(fn, cls=Group)
+        if not isinstance(cmd, click.core.Group):
+            raise ValueError(f"generated src command was not of expected type Group: {type(cmd)}")
+        cmd.name = self.cli_cmd_name
+        cmd.short_help = "v2"
+        cmd.invoke_without_command = True
+        self.add_options(cmd)
+
+        # TODO remove after v1 no longer supported
+        cmd.params.append(
+            click.Option(
+                ["--output-dir"],
+                required=False,
+                type=str,
+                help="Local path to write partitioned output to",
+            )
+        )
+        return cmd

unstructured_ingest/v2/cli/cli.py

@@ -0,0 +1,24 @@
+import click
+
+from unstructured_ingest.v2.cli.cmds import dest, src
+
+
+@click.group()
+def ingest():
+    pass
+
+
+def get_cmd() -> click.Command:
+    """Construct and return a Click command object representing the main command for the CLI.
+
+    This function adds all dest_subcommand(s) to each src_subcommand, and adds all of those
+    to the main command as nested subcommands.
+    """
+    cmd = ingest
+    # Add all subcommands
+    for src_subcommand in src:
+        # Add all destination subcommands
+        for dest_subcommand in dest:
+            src_subcommand.add_command(dest_subcommand)
+        cmd.add_command(src_subcommand)
+    return cmd

unstructured_ingest/v2/cli/cmds.py

@@ -0,0 +1,14 @@
+import click
+
+from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
+from unstructured_ingest.v2.processes.connector_registry import (
+    destination_registry,
+    source_registry,
+)
+
+src_cmds = [SrcCmd(cmd_name=k, registry_entry=v) for k, v in source_registry.items()]
+dest_cmds = [DestCmd(cmd_name=k, registry_entry=v) for k, v in destination_registry.items()]
+
+src: list[click.Group] = [v.get_cmd() for v in src_cmds]
+
+dest: list[click.Command] = [v.get_cmd() for v in dest_cmds]

unstructured_ingest/v2/cli/utils/click.py

@@ -0,0 +1,237 @@
+import json
+import os.path
+from datetime import date, datetime
+from gettext import gettext, ngettext
+from gettext import gettext as _
+from pathlib import Path
+from typing import Any, Optional, Type, TypeVar, Union
+
+import click
+from pydantic import BaseModel, ConfigDict, Secret, TypeAdapter, ValidationError
+
+
+def conform_click_options(options: dict):
+    # Click sets all multiple fields as tuple, this needs to be updated to list
+    for k, v in options.items():
+        if isinstance(v, tuple):
+            options[k] = list(v)
+
+
+class Dict(click.ParamType):
+    name = "dict"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            if isinstance(value, dict):
+                return value
+            if isinstance(value, Path) and value.is_file():
+                with value.open() as f:
+                    return json.load(f)
+            if isinstance(value, str):
+                return json.loads(value)
+        except json.JSONDecodeError:
+            self.fail(
+                gettext(
+                    "{value} is not a valid json value.",
+                ).format(value=value),
+                param,
+                ctx,
+            )
+
+
+class FileOrJson(click.ParamType):
+    name = "file-or-json"
+
+    def __init__(self, allow_raw_str: bool = False):
+        self.allow_raw_str = allow_raw_str
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        # check if valid file
+        full_path = os.path.abspath(os.path.expanduser(value))
+        if os.path.isfile(full_path):
+            return str(Path(full_path).resolve())
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except json.JSONDecodeError:
+                if self.allow_raw_str:
+                    return value
+        self.fail(
+            gettext(
+                "{value} is neither a valid json string nor an existing filepath.",
+            ).format(value=value),
+            param,
+            ctx,
+        )
+
+
+class DelimitedString(click.ParamType):
+    name = "delimited-string"
+
+    def __init__(self, delimiter: str = ",", choices: Optional[list[str]] = None):
+        self.choices = choices if choices else []
+        self.delimiter = delimiter
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        # In case a list is provided as the default, will not break
+        if isinstance(value, list):
+            split = [str(v).strip() for v in value]
+        else:
+            split = [v.strip() for v in value.split(self.delimiter)]
+        if not self.choices:
+            return split
+        choices_str = ", ".join(map(repr, self.choices))
+        for s in split:
+            if s not in self.choices:
+                self.fail(
+                    ngettext(
+                        "{value!r} is not {choice}.",
+                        "{value!r} is not one of {choices}.",
+                        len(self.choices),
+                    ).format(value=s, choice=choices_str, choices=choices_str),
+                    param,
+                    ctx,
+                )
+        return split
+
+
+class PydanticDateTime(click.ParamType):
+    name = "datetime"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return TypeAdapter(datetime).validate_strings(value)
+        except ValidationError:
+            self.fail(f"{value} is not a valid datetime", param, ctx)
+
+
+class PydanticDate(click.ParamType):
+    name = "date"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return TypeAdapter(date).validate_strings(value)
+        except ValidationError:
+            self.fail(f"{value} is not a valid date", param, ctx)
+
+
+BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
+
+
+def unwrap_optional(val: Any) -> tuple[Any, bool]:
+    if (
+        hasattr(val, "__origin__")
+        and hasattr(val, "__args__")
+        and val.__origin__ is Union
+        and len(val.__args__) == 2
+        and type(None) in val.__args__
+    ):
+        args = val.__args__
+        args = [a for a in args if a is not None]
+        return args[0], True
+    return val, False
+
+
+def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
+    fields = config.model_fields
+    config.model_config = ConfigDict(extra="ignore")
+    field_names = [v.alias or k for k, v in fields.items()]
+    data = {k: v for k, v in flat_data.items() if k in field_names and v is not None}
+    if access_config := fields.get("access_config"):
+        access_config_type = access_config.annotation
+        access_config_type, is_optional = unwrap_optional(access_config_type)
+        # Check if raw type is wrapped by a secret
+        if (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Secret
+        ):
+            ac_subtypes = access_config_type.__args__
+            ac_fields = ac_subtypes[0].model_fields
+        elif issubclass(access_config_type, BaseModel):
+            ac_fields = access_config_type.model_fields
+        else:
+            raise TypeError(f"Unrecognized access_config type: {access_config_type}")
+        ac_field_names = [v.alias or k for k, v in ac_fields.items()]
+        access_config_data = {
+            k: v for k, v in flat_data.items() if k in ac_field_names and v is not None
+        }
+        if not access_config_data and is_optional:
+            data["access_config"] = None
+        else:
+            data["access_config"] = access_config_data
+    return config.model_validate(obj=data)
+
+
+class Group(click.Group):
+    def parse_args(self, ctx, args):
+        """
+        This allows for subcommands to be called with the --help flag without breaking
+        if parent command is missing any of its required parameters
+        """
+        try:
+            return super().parse_args(ctx, args)
+        except click.MissingParameter:
+            if "--help" not in args:
+                raise
+            # remove the required params so that help can display
+            for param in self.params:
+                param.required = False
+            return super().parse_args(ctx, args)
+
+    def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
+        """
+        Copy of the original click.Group format_commands() method but replacing
+        'Commands' -> 'Destinations'
+        """
+        commands = []
+        for subcommand in self.list_commands(ctx):
+            cmd = self.get_command(ctx, subcommand)
+            # What is this, the tool lied about a command.  Ignore it
+            if cmd is None:
+                continue
+            if cmd.hidden:
+                continue
+
+            commands.append((subcommand, cmd))
+
+        # allow for 3 times the default spacing
+        if len(commands):
+            if formatter.width:
+                limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands)
+            else:
+                limit = -6 - max(len(cmd[0]) for cmd in commands)
+
+            rows = []
+            for subcommand, cmd in commands:
+                help = cmd.get_short_help_str(limit)
+                rows.append((subcommand, help))
+
+            if rows:
+                with formatter.section(_("Destinations")):
+                    formatter.write_dl(rows)

unstructured_ingest/v2/cli/utils/model_conversion.py

@@ -0,0 +1,222 @@
+import contextlib
+import datetime
+from collections import Counter
+from enum import EnumMeta
+from pathlib import Path
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Literal,
+    Optional,
+    Type,
+    TypedDict,
+    Union,
+    get_args,
+    get_origin,
+)
+from uuid import UUID
+
+import click
+from annotated_types import Ge, Gt, Le, Lt, SupportsGe, SupportsGt, SupportsLe, SupportsLt
+from click import Option
+from pydantic import BaseModel, Secret, SecretStr
+from pydantic.fields import FieldInfo
+from pydantic.types import _SecretBase
+from pydantic_core import PydanticUndefined
+
+from unstructured_ingest.v2.cli.utils.click import (
+    DelimitedString,
+    Dict,
+    PydanticDate,
+    PydanticDateTime,
+)
+
+NoneType = type(None)
+
+
+class _RangeDict(TypedDict, total=False):
+    """Represent arguments to `click.IntRange` or `click.FloatRange`."""
+
+    max: Union[SupportsLt, SupportsLe]
+    min: Union[SupportsGt, SupportsGe]
+    max_open: bool
+    min_open: bool
+
+
+def get_range_from_metadata(metadata: list[Any]) -> _RangeDict:
+    range_args: _RangeDict = {}
+    for constraint in metadata:
+        if isinstance(constraint, Le):
+            range_args["max"] = constraint.le
+            range_args["max_open"] = False
+        if isinstance(constraint, Lt):
+            range_args["max"] = constraint.lt
+            range_args["max_open"] = True
+        if isinstance(constraint, Ge):
+            range_args["min"] = constraint.ge
+            range_args["min_open"] = False
+        if isinstance(constraint, Gt):
+            range_args["min"] = constraint.gt
+            range_args["min_open"] = True
+    return range_args
+
+
+def is_boolean_flag(field_info: FieldInfo) -> bool:
+    annotation = field_info.annotation
+    raw_annotation = get_raw_type(annotation)
+    return raw_annotation is bool
+
+
+def get_raw_type(val: Any) -> Any:
+    field_args = get_args(val)
+    field_origin = get_origin(val)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if val is SecretStr:
+        return str
+    return val
+
+
+def get_default_value_from_field(field: FieldInfo) -> Optional[Union[Any, Callable[[], Any]]]:
+    if field.default is not PydanticUndefined:
+        return field.default
+    elif field.default_factory is not None:
+        return field.default_factory
+    return None
+
+
+def get_option_name(field_name: str, field_info: FieldInfo) -> str:
+    field_name = field_info.alias or field_name
+    if field_name.startswith("--"):
+        field_name = field_name[2:]
+    field_name = field_name.lower().replace("_", "-")
+    if is_boolean_flag(field_info):
+        return f"--{field_name}/--no-{field_name}"
+    return f"--{field_name}"
+
+
+def get_numerical_type(field: FieldInfo) -> click.ParamType:
+    range_args = get_range_from_metadata(field.metadata)
+    if field.annotation is int:
+        if range_args:
+            return click.IntRange(**range_args)  # type: ignore[arg-type]
+        return click.INT
+    # Non-integer numerical types default to float
+    if range_args:
+        return click.FloatRange(**range_args)  # type: ignore[arg-type]
+    return click.FLOAT
+
+
+def get_type_from_annotation(field_type: Any) -> click.ParamType:
+    field_origin = get_origin(field_type)
+    field_args = get_args(field_type)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is Annotated:
+        field_origin = field_args[0]
+        field_metadata = field_args[1]
+        if isinstance(field_metadata, click.ParamType):
+            return field_metadata
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is list and len(field_args) == 1 and field_args[0] is str:
+        return DelimitedString()
+    if field_type is SecretStr:
+        return click.STRING
+    if dict in [field_type, field_origin]:
+        return Dict()
+    if field_type is str:
+        return click.STRING
+    if field_type is bool:
+        return click.BOOL
+    if field_type is UUID:
+        return click.UUID
+    if field_type is Path:
+        return click.Path(path_type=Path)
+    if field_type is datetime.datetime:
+        return PydanticDateTime()
+    if field_type is datetime.date:
+        return PydanticDate()
+    if field_origin is Literal:
+        return click.Choice(field_args)
+    if isinstance(field_type, EnumMeta):
+        values = [i.value for i in field_type]
+        return click.Choice(values)
+    raise TypeError(f"Unexpected field type: {field_type}")
+
+
+def _get_type_from_field(field: FieldInfo) -> click.ParamType:
+    raw_field_type = get_raw_type(field.annotation)
+
+    if raw_field_type in (int, float):
+        return get_numerical_type(field)
+    return get_type_from_annotation(field_type=field.annotation)
+
+
+def get_option_from_field(option_name: str, field_info: FieldInfo) -> Option:
+    param_decls = [option_name]
+    help_text = field_info.description or ""
+    if examples := field_info.examples:
+        help_text += f" [Examples: {', '.join(examples)}]"
+    option_kwargs = {
+        "type": _get_type_from_field(field_info),
+        "default": get_default_value_from_field(field_info),
+        "required": field_info.is_required(),
+        "help": str(help_text),
+        "is_flag": is_boolean_flag(field_info),
+        "show_default": field_info.default is not PydanticUndefined,
+    }
+    return click.Option(param_decls=param_decls, **option_kwargs)
+
+
+def is_subclass(x: Any, y: Any) -> bool:
+    with contextlib.suppress(TypeError):
+        return issubclass(x, y)
+
+    return False
+
+
+def post_check(options: list[Option]):
+    option_names = [option.name for option in options]
+    duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+    if duplicate_names:
+        raise ValueError(
+            "the following field name were reused, all must be unique: {}".format(
+                ", ".join(duplicate_names)
+            )
+        )
+
+
+def is_secret(value: Any) -> bool:
+    # Case Secret[int]
+    if hasattr(value, "__origin__") and hasattr(value, "__args__"):
+        origin = value.__origin__
+        return is_subclass(origin, _SecretBase)
+    # Case SecretStr
+    return is_subclass(value, _SecretBase)
+
+
+def options_from_base_model(model: Union[BaseModel, Type[BaseModel]]) -> list[Option]:
+    options = []
+    model_fields = model.model_fields
+    for field_name, field_info in model_fields.items():
+        if field_info.init is False:
+            continue
+        option_name = get_option_name(field_name=field_name, field_info=field_info)
+        raw_annotation = get_raw_type(field_info.annotation)
+        if is_subclass(raw_annotation, BaseModel):
+            options.extend(options_from_base_model(model=raw_annotation))
+        else:
+            if is_secret(field_info.annotation):
+                field_info.description = f"[sensitive] {field_info.description}"
+            options.append(get_option_from_field(option_name=option_name, field_info=field_info))
+
+    post_check(options=options)
+    return options

unstructured_ingest/v2/constants.py

@@ -0,0 +1,2 @@
+# Used to append to metadata for uploaders that store element-level data
+RECORD_ID_LABEL = "record_id"

unstructured_ingest/v2/errors.py

@@ -0,0 +1,18 @@
+class UserError(Exception):
+    pass
+
+
+class UserAuthError(UserError):
+    pass
+
+
+class RateLimitError(UserError):
+    pass
+
+
+class QuotaError(UserError):
+    pass
+
+
+class ProviderError(Exception):
+    pass

unstructured_ingest/v2/interfaces/__init__.py

@@ -0,0 +1,32 @@
+from .connector import AccessConfig, BaseConnector, ConnectionConfig
+from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
+from .file_data import BatchFileData, BatchItem, FileData, FileDataSourceMetadata, SourceIdentifiers
+from .indexer import Indexer, IndexerConfig
+from .process import BaseProcess
+from .processor import ProcessorConfig
+from .upload_stager import UploadStager, UploadStagerConfig
+from .uploader import UploadContent, Uploader, UploaderConfig
+
+__all__ = [
+    "DownloadResponse",
+    "download_responses",
+    "Downloader",
+    "DownloaderConfig",
+    "FileData",
+    "Indexer",
+    "IndexerConfig",
+    "BaseProcess",
+    "ProcessorConfig",
+    "UploadStager",
+    "UploadStagerConfig",
+    "Uploader",
+    "UploaderConfig",
+    "SourceIdentifiers",
+    "UploadContent",
+    "AccessConfig",
+    "ConnectionConfig",
+    "BaseConnector",
+    "FileDataSourceMetadata",
+    "BatchFileData",
+    "BatchItem",
+]

unstructured_ingest/v2/interfaces/connector.py

@@ -0,0 +1,50 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import Any, TypeVar, Union
+
+from pydantic import BaseModel, Secret, model_validator
+from pydantic.types import _SecretBase
+
+
+class AccessConfig(BaseModel):
+    """Meant to designate holding any sensitive information associated with other configs
+    and also for access specific configs."""
+
+
+AccessConfigT = TypeVar("AccessConfigT", bound=AccessConfig)
+
+
+class ConnectionConfig(BaseModel):
+    access_config: Secret[AccessConfigT]
+
+    def get_access_config(self) -> dict[str, Any]:
+        if not self.access_config:
+            return {}
+        return self.access_config.get_secret_value().model_dump()
+
+    @model_validator(mode="after")
+    def check_access_config(self):
+        access_config = self.access_config
+        if self._is_access_config_optional() and access_config is None:
+            return self
+        if not isinstance(access_config, _SecretBase):
+            raise ValueError("access_config must be an instance of SecretBase")
+        return self
+
+    def _is_access_config_optional(self) -> bool:
+        access_config_type = self.model_fields["access_config"].annotation
+        return (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Union
+            and len(access_config_type.__args__) == 2
+            and type(None) in access_config_type.__args__
+        )
+
+
+ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)
+
+
+@dataclass
+class BaseConnector(ABC):
+    connection_config: ConnectionConfigT