unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/cli/utils/click.py

@@ -0,0 +1,237 @@
+import json
+import os.path
+from datetime import date, datetime
+from gettext import gettext, ngettext
+from gettext import gettext as _
+from pathlib import Path
+from typing import Any, Optional, Type, TypeVar, Union
+
+import click
+from pydantic import BaseModel, ConfigDict, Secret, TypeAdapter, ValidationError
+
+
+def conform_click_options(options: dict):
+    # Click sets all multiple fields as tuple, this needs to be updated to list
+    for k, v in options.items():
+        if isinstance(v, tuple):
+            options[k] = list(v)
+
+
+class Dict(click.ParamType):
+    name = "dict"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            if isinstance(value, dict):
+                return value
+            if isinstance(value, Path) and value.is_file():
+                with value.open() as f:
+                    return json.load(f)
+            if isinstance(value, str):
+                return json.loads(value)
+        except json.JSONDecodeError:
+            self.fail(
+                gettext(
+                    "{value} is not a valid json value.",
+                ).format(value=value),
+                param,
+                ctx,
+            )
+
+
+class FileOrJson(click.ParamType):
+    name = "file-or-json"
+
+    def __init__(self, allow_raw_str: bool = False):
+        self.allow_raw_str = allow_raw_str
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        # check if valid file
+        full_path = os.path.abspath(os.path.expanduser(value))
+        if os.path.isfile(full_path):
+            return str(Path(full_path).resolve())
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except json.JSONDecodeError:
+                if self.allow_raw_str:
+                    return value
+        self.fail(
+            gettext(
+                "{value} is neither a valid json string nor an existing filepath.",
+            ).format(value=value),
+            param,
+            ctx,
+        )
+
+
+class DelimitedString(click.ParamType):
+    name = "delimited-string"
+
+    def __init__(self, delimiter: str = ",", choices: Optional[list[str]] = None):
+        self.choices = choices if choices else []
+        self.delimiter = delimiter
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        # In case a list is provided as the default, will not break
+        if isinstance(value, list):
+            split = [str(v).strip() for v in value]
+        else:
+            split = [v.strip() for v in value.split(self.delimiter)]
+        if not self.choices:
+            return split
+        choices_str = ", ".join(map(repr, self.choices))
+        for s in split:
+            if s not in self.choices:
+                self.fail(
+                    ngettext(
+                        "{value!r} is not {choice}.",
+                        "{value!r} is not one of {choices}.",
+                        len(self.choices),
+                    ).format(value=s, choice=choices_str, choices=choices_str),
+                    param,
+                    ctx,
+                )
+        return split
+
+
+class PydanticDateTime(click.ParamType):
+    name = "datetime"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return TypeAdapter(datetime).validate_strings(value)
+        except ValidationError:
+            self.fail(f"{value} is not a valid datetime", param, ctx)
+
+
+class PydanticDate(click.ParamType):
+    name = "date"
+
+    def convert(
+        self,
+        value: Any,
+        param: Optional[click.Parameter] = None,
+        ctx: Optional[click.Context] = None,
+    ) -> Any:
+        try:
+            return TypeAdapter(date).validate_strings(value)
+        except ValidationError:
+            self.fail(f"{value} is not a valid date", param, ctx)
+
+
+BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
+
+
+def unwrap_optional(val: Any) -> tuple[Any, bool]:
+    if (
+        hasattr(val, "__origin__")
+        and hasattr(val, "__args__")
+        and val.__origin__ is Union
+        and len(val.__args__) == 2
+        and type(None) in val.__args__
+    ):
+        args = val.__args__
+        args = [a for a in args if a is not None]
+        return args[0], True
+    return val, False
+
+
+def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
+    fields = config.model_fields
+    config.model_config = ConfigDict(extra="ignore")
+    field_names = [v.alias or k for k, v in fields.items()]
+    data = {k: v for k, v in flat_data.items() if k in field_names and v is not None}
+    if access_config := fields.get("access_config"):
+        access_config_type = access_config.annotation
+        access_config_type, is_optional = unwrap_optional(access_config_type)
+        # Check if raw type is wrapped by a secret
+        if (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Secret
+        ):
+            ac_subtypes = access_config_type.__args__
+            ac_fields = ac_subtypes[0].model_fields
+        elif issubclass(access_config_type, BaseModel):
+            ac_fields = access_config_type.model_fields
+        else:
+            raise TypeError(f"Unrecognized access_config type: {access_config_type}")
+        ac_field_names = [v.alias or k for k, v in ac_fields.items()]
+        access_config_data = {
+            k: v for k, v in flat_data.items() if k in ac_field_names and v is not None
+        }
+        if not access_config_data and is_optional:
+            data["access_config"] = None
+        else:
+            data["access_config"] = access_config_data
+    return config.model_validate(obj=data)
+
+
+class Group(click.Group):
+    def parse_args(self, ctx, args):
+        """
+        This allows for subcommands to be called with the --help flag without breaking
+        if parent command is missing any of its required parameters
+        """
+        try:
+            return super().parse_args(ctx, args)
+        except click.MissingParameter:
+            if "--help" not in args:
+                raise
+            # remove the required params so that help can display
+            for param in self.params:
+                param.required = False
+            return super().parse_args(ctx, args)
+
+    def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
+        """
+        Copy of the original click.Group format_commands() method but replacing
+        'Commands' -> 'Destinations'
+        """
+        commands = []
+        for subcommand in self.list_commands(ctx):
+            cmd = self.get_command(ctx, subcommand)
+            # What is this, the tool lied about a command.  Ignore it
+            if cmd is None:
+                continue
+            if cmd.hidden:
+                continue
+
+            commands.append((subcommand, cmd))
+
+        # allow for 3 times the default spacing
+        if len(commands):
+            if formatter.width:
+                limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands)
+            else:
+                limit = -6 - max(len(cmd[0]) for cmd in commands)
+
+            rows = []
+            for subcommand, cmd in commands:
+                help = cmd.get_short_help_str(limit)
+                rows.append((subcommand, help))
+
+            if rows:
+                with formatter.section(_("Destinations")):
+                    formatter.write_dl(rows)

unstructured_ingest/cli/utils/model_conversion.py

@@ -0,0 +1,222 @@
+import contextlib
+import datetime
+from collections import Counter
+from enum import EnumMeta
+from pathlib import Path
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Literal,
+    Optional,
+    Type,
+    TypedDict,
+    Union,
+    get_args,
+    get_origin,
+)
+from uuid import UUID
+
+import click
+from annotated_types import Ge, Gt, Le, Lt, SupportsGe, SupportsGt, SupportsLe, SupportsLt
+from click import Option
+from pydantic import BaseModel, Secret, SecretStr
+from pydantic.fields import FieldInfo
+from pydantic.types import _SecretBase
+from pydantic_core import PydanticUndefined
+
+from unstructured_ingest.cli.utils.click import (
+    DelimitedString,
+    Dict,
+    PydanticDate,
+    PydanticDateTime,
+)
+
+NoneType = type(None)
+
+
+class _RangeDict(TypedDict, total=False):
+    """Represent arguments to `click.IntRange` or `click.FloatRange`."""
+
+    max: Union[SupportsLt, SupportsLe]
+    min: Union[SupportsGt, SupportsGe]
+    max_open: bool
+    min_open: bool
+
+
+def get_range_from_metadata(metadata: list[Any]) -> _RangeDict:
+    range_args: _RangeDict = {}
+    for constraint in metadata:
+        if isinstance(constraint, Le):
+            range_args["max"] = constraint.le
+            range_args["max_open"] = False
+        if isinstance(constraint, Lt):
+            range_args["max"] = constraint.lt
+            range_args["max_open"] = True
+        if isinstance(constraint, Ge):
+            range_args["min"] = constraint.ge
+            range_args["min_open"] = False
+        if isinstance(constraint, Gt):
+            range_args["min"] = constraint.gt
+            range_args["min_open"] = True
+    return range_args
+
+
+def is_boolean_flag(field_info: FieldInfo) -> bool:
+    annotation = field_info.annotation
+    raw_annotation = get_raw_type(annotation)
+    return raw_annotation is bool
+
+
+def get_raw_type(val: Any) -> Any:
+    field_args = get_args(val)
+    field_origin = get_origin(val)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if val is SecretStr:
+        return str
+    return val
+
+
+def get_default_value_from_field(field: FieldInfo) -> Optional[Union[Any, Callable[[], Any]]]:
+    if field.default is not PydanticUndefined:
+        return field.default
+    elif field.default_factory is not None:
+        return field.default_factory
+    return None
+
+
+def get_option_name(field_name: str, field_info: FieldInfo) -> str:
+    field_name = field_info.alias or field_name
+    if field_name.startswith("--"):
+        field_name = field_name[2:]
+    field_name = field_name.lower().replace("_", "-")
+    if is_boolean_flag(field_info):
+        return f"--{field_name}/--no-{field_name}"
+    return f"--{field_name}"
+
+
+def get_numerical_type(field: FieldInfo) -> click.ParamType:
+    range_args = get_range_from_metadata(field.metadata)
+    if field.annotation is int:
+        if range_args:
+            return click.IntRange(**range_args)  # type: ignore[arg-type]
+        return click.INT
+    # Non-integer numerical data_types default to float
+    if range_args:
+        return click.FloatRange(**range_args)  # type: ignore[arg-type]
+    return click.FLOAT
+
+
+def get_type_from_annotation(field_type: Any) -> click.ParamType:
+    field_origin = get_origin(field_type)
+    field_args = get_args(field_type)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is Annotated:
+        field_origin = field_args[0]
+        field_metadata = field_args[1]
+        if isinstance(field_metadata, click.ParamType):
+            return field_metadata
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is list and len(field_args) == 1 and field_args[0] is str:
+        return DelimitedString()
+    if field_type is SecretStr:
+        return click.STRING
+    if dict in [field_type, field_origin]:
+        return Dict()
+    if field_type is str:
+        return click.STRING
+    if field_type is bool:
+        return click.BOOL
+    if field_type is UUID:
+        return click.UUID
+    if field_type is Path:
+        return click.Path(path_type=Path)
+    if field_type is datetime.datetime:
+        return PydanticDateTime()
+    if field_type is datetime.date:
+        return PydanticDate()
+    if field_origin is Literal:
+        return click.Choice(field_args)
+    if isinstance(field_type, EnumMeta):
+        values = [i.value for i in field_type]
+        return click.Choice(values)
+    raise TypeError(f"Unexpected field type: {field_type}")
+
+
+def _get_type_from_field(field: FieldInfo) -> click.ParamType:
+    raw_field_type = get_raw_type(field.annotation)
+
+    if raw_field_type in (int, float):
+        return get_numerical_type(field)
+    return get_type_from_annotation(field_type=field.annotation)
+
+
+def get_option_from_field(option_name: str, field_info: FieldInfo) -> Option:
+    param_decls = [option_name]
+    help_text = field_info.description or ""
+    if examples := field_info.examples:
+        help_text += f" [Examples: {', '.join(examples)}]"
+    option_kwargs = {
+        "type": _get_type_from_field(field_info),
+        "default": get_default_value_from_field(field_info),
+        "required": field_info.is_required(),
+        "help": str(help_text),
+        "is_flag": is_boolean_flag(field_info),
+        "show_default": field_info.default is not PydanticUndefined,
+    }
+    return click.Option(param_decls=param_decls, **option_kwargs)
+
+
+def is_subclass(x: Any, y: Any) -> bool:
+    with contextlib.suppress(TypeError):
+        return issubclass(x, y)
+
+    return False
+
+
+def post_check(options: list[Option], name: str):
+    option_names = [option.name for option in options]
+    duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+    if duplicate_names:
+        raise ValueError(
+            "[{}] the following field name were reused, all must be unique: {}".format(
+                name, ", ".join(duplicate_names)
+            )
+        )
+
+
+def is_secret(value: Any) -> bool:
+    # Case Secret[int]
+    if hasattr(value, "__origin__") and hasattr(value, "__args__"):
+        origin = value.__origin__
+        return is_subclass(origin, _SecretBase)
+    # Case SecretStr
+    return is_subclass(value, _SecretBase)
+
+
+def options_from_base_model(model: Union[BaseModel, Type[BaseModel]]) -> list[Option]:
+    options = []
+    model_fields = model.model_fields
+    for field_name, field_info in model_fields.items():
+        if field_info.init is False:
+            continue
+        option_name = get_option_name(field_name=field_name, field_info=field_info)
+        raw_annotation = get_raw_type(field_info.annotation)
+        if is_subclass(raw_annotation, BaseModel):
+            options.extend(options_from_base_model(model=raw_annotation))
+        else:
+            if is_secret(field_info.annotation):
+                field_info.description = f"[sensitive] {field_info.description}"
+            options.append(get_option_from_field(option_name=option_name, field_info=field_info))
+
+    post_check(options=options, name=model.__name__)
+    return options

unstructured_ingest/data_types/entities.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel, Field
+
+
+class Entity(BaseModel):
+    type: str
+    entity: str
+
+
+class EntityRelationship(BaseModel):
+    to: str
+    from_: str = Field(..., alias="from")
+    relationship: str
+
+
+class EntitiesData(BaseModel):
+    items: list[Entity] = Field(default_factory=list)
+    relationships: list[EntityRelationship] = Field(default_factory=list)

unstructured_ingest/data_types/file_data.py

@@ -0,0 +1,116 @@
+import json
+from pathlib import Path
+from typing import Any, Optional
+from uuid import NAMESPACE_DNS, uuid5
+
+from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator
+
+from unstructured_ingest.logger import logger
+
+
+class SourceIdentifiers(BaseModel):
+    filename: str
+    fullpath: str
+    rel_path: Optional[str] = None
+
+    @property
+    def filename_stem(self) -> str:
+        return Path(self.filename).stem
+
+    @property
+    def relative_path(self) -> str:
+        return self.rel_path or self.fullpath
+
+
+class FileDataSourceMetadata(BaseModel):
+    url: Optional[str] = None
+    version: Optional[str] = None
+    record_locator: Optional[dict[str, Any]] = None
+    date_created: Optional[str] = None
+    date_modified: Optional[str] = None
+    date_processed: Optional[str] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
+    filesize_bytes: Optional[int] = None
+
+
+class FileData(BaseModel):
+    identifier: str
+    connector_type: str
+    source_identifiers: SourceIdentifiers
+    metadata: FileDataSourceMetadata = Field(default_factory=lambda: FileDataSourceMetadata())
+    additional_metadata: dict[str, Any] = Field(default_factory=dict)
+    reprocess: bool = False
+    local_download_path: Optional[str] = None
+    display_name: Optional[str] = None
+
+    @classmethod
+    def from_file(cls, path: str) -> "FileData":
+        path = Path(path).resolve()
+        if not path.exists() or not path.is_file():
+            raise ValueError(f"file path not valid: {path}")
+        with open(str(path.resolve()), "rb") as f:
+            file_data_dict = json.load(f)
+        file_data = cls.model_validate(file_data_dict)
+        return file_data
+
+    @classmethod
+    def cast(cls, file_data: "FileData", **kwargs) -> "FileData":
+        file_data_dict = file_data.model_dump()
+        return cls.model_validate(file_data_dict, **kwargs)
+
+    def to_file(self, path: str) -> None:
+        path = Path(path).resolve()
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(str(path.resolve()), "w") as f:
+            json.dump(self.model_dump(), f, indent=2)
+
+
+class BatchItem(BaseModel):
+    identifier: str
+    version: Optional[str] = None
+
+
+class BatchFileData(FileData):
+    identifier: str = Field(init=False)
+    batch_items: list[BatchItem]
+    source_identifiers: Optional[SourceIdentifiers] = None
+
+    @field_validator("batch_items")
+    @classmethod
+    def check_batch_items(cls, v: list[BatchItem]) -> list[BatchItem]:
+        if not v:
+            raise ValueError("batch items cannot be empty")
+        all_identifiers = [item.identifier for item in v]
+        if len(all_identifiers) != len(set(all_identifiers)):
+            raise ValueError(f"duplicate identifiers: {all_identifiers}")
+        sorted_batch_items = sorted(v, key=lambda item: item.identifier)
+        return sorted_batch_items
+
+    @model_validator(mode="before")
+    @classmethod
+    def populate_identifier(cls, data: Any) -> Any:
+        if isinstance(data, dict) and "identifier" not in data:
+            batch_items = data["batch_items"]
+            identifier_data = json.dumps(
+                {item.identifier: item.version for item in batch_items}, sort_keys=True
+            )
+            data["identifier"] = str(uuid5(NAMESPACE_DNS, str(identifier_data)))
+        return data
+
+
+def file_data_from_file(path: str) -> FileData:
+    try:
+        return BatchFileData.from_file(path=path)
+    except ValidationError:
+        logger.debug(f"{path} not detected as batch file data")
+
+    return FileData.from_file(path=path)
+
+
+def file_data_from_dict(data: dict) -> FileData:
+    try:
+        return BatchFileData.model_validate(data)
+    except ValidationError:
+        logger.debug(f"{data} not valid for batch file data")
+
+    return FileData.model_validate(data)

unstructured_ingest/embed/azure_openai.py

@@ -0,0 +1,63 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pydantic import Field
+
+from unstructured_ingest.embed.openai import (
+    AsyncOpenAIEmbeddingEncoder,
+    OpenAIEmbeddingConfig,
+    OpenAIEmbeddingEncoder,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.tls import ssl_context_with_optional_ca_override
+
+if TYPE_CHECKING:
+    from openai import AsyncAzureOpenAI, AzureOpenAI
+
+
+class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
+    api_version: str = Field(description="Azure API version", default="2024-06-01")
+    azure_endpoint: str = Field(description="Azure endpoint")
+    embedder_model_name: str = Field(
+        default="text-embedding-ada-002", alias="model_name", description="Azure OpenAI model name"
+    )
+
+    @requires_dependencies(["openai"], extras="openai")
+    def get_client(self) -> "AzureOpenAI":
+        from openai import AzureOpenAI, DefaultHttpxClient
+
+        client = DefaultHttpxClient(verify=ssl_context_with_optional_ca_override())
+        return AzureOpenAI(
+            http_client=client,
+            api_key=self.api_key.get_secret_value(),
+            api_version=self.api_version,
+            azure_endpoint=self.azure_endpoint,
+        )
+
+    @requires_dependencies(["openai"], extras="openai")
+    def get_async_client(self) -> "AsyncAzureOpenAI":
+        from openai import AsyncAzureOpenAI, DefaultAsyncHttpxClient
+
+        client = DefaultAsyncHttpxClient(verify=ssl_context_with_optional_ca_override())
+        return AsyncAzureOpenAI(
+            http_client=client,
+            api_key=self.api_key.get_secret_value(),
+            api_version=self.api_version,
+            azure_endpoint=self.azure_endpoint,
+        )
+
+
+@dataclass
+class AzureOpenAIEmbeddingEncoder(OpenAIEmbeddingEncoder):
+    config: AzureOpenAIEmbeddingConfig
+
+    def get_client(self) -> "AzureOpenAI":
+        return self.config.get_client()
+
+
+@dataclass
+class AsyncAzureOpenAIEmbeddingEncoder(AsyncOpenAIEmbeddingEncoder):
+    config: AzureOpenAIEmbeddingConfig
+
+    def get_client(self) -> "AsyncAzureOpenAI":
+        return self.config.get_async_client()