unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/cli/{utils.py → utils/click.py}

@@ -1,16 +1,12 @@
 import json
 import os.path
-import sys
-from dataclasses import fields, is_dataclass
 from gettext import gettext, ngettext
 from gettext import gettext as _
 from pathlib import Path
-from typing import Any, ForwardRef, Optional, Type, TypeVar, Union, get_args, get_origin
+from typing import Any, Optional, Type, TypeVar
 
 import click
-
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
-from unstructured_ingest.v2.logger import logger
+from pydantic import BaseModel, ConfigDict, Secret
 
 
 def conform_click_options(options: dict):
@@ -30,7 +26,13 @@ class Dict(click.ParamType):
         ctx: Optional[click.Context] = None,
     ) -> Any:
         try:
-            return json.loads(value)
+            if isinstance(value, dict):
+                return value
+            if isinstance(value, Path) and value.is_file():
+                with value.open() as f:
+                    return json.load(f)
+            if isinstance(value, str):
+                return json.loads(value)
         except json.JSONDecodeError:
             self.fail(
                 gettext(
@@ -107,86 +109,33 @@ class DelimitedString(click.ParamType):
         return split
 
 
-EnhancedDataClassJsonMixinT = TypeVar(
-    "EnhancedDataClassJsonMixinT", bound=EnhancedDataClassJsonMixin
-)
-
-
-def extract_config(
-    flat_data: dict, config: Type[EnhancedDataClassJsonMixinT]
-) -> EnhancedDataClassJsonMixinT:
-    """
-    To be able to extract a nested dataclass from a flat dictionary (as in one coming
-    from a click-based options input), the config class is dynamically looked through for
-    nested dataclass fields and new nested dictionaries are created to conform to the
-    shape the overall class expects when parsing from a dict. During the process, this will create
-    copies of the original dictionary to avoid pruning fields but this isn't a
-    problem since the `from_dict()` method ignores unneeded values.
-
-    Not handling more complex edge cases for now such as nested types i.e Union[List[List[...]]]
-    """
-
-    def conform_dict(inner_d: dict, inner_config: Type[EnhancedDataClassJsonMixinT]):
-        # Catch edge cases (i.e. Dict[str, ...]) where underlying type is not a concrete Class,
-        # causing 'issubclass() arg 1 must be a class' errors, return False
-        def is_subclass(instance, class_type) -> bool:
-            try:
-                return issubclass(instance, class_type)
-            except Exception:
-                return False
-
-        dd = inner_d.copy()
-        for field in fields(inner_config):
-            f_type = field.type
-            # typing can be defined using a string, in which case it needs to be resolved
-            # to the actual type. following logic is cherry picked from the typing
-            # get_type_hints() since type resolution can be expensive, only do it
-            # when the type is a string
-            if isinstance(f_type, str):
-                try:
-                    base_globals = sys.modules[inner_config.__module__].__dict__
-                    for_ref = ForwardRef(f_type, is_argument=False, is_class=True)
-                    f_type = for_ref._evaluate(
-                        globalns=base_globals, localns=None, recursive_guard=frozenset()
-                    )
-                except NameError as e:
-                    logger.warning(f"couldn't resolve type {f_type}: {e}")
-            # Handle the case where the type of a value if a Union (possibly optional)
-            if get_origin(f_type) is Union:
-                union_values = get_args(f_type)
-                # handle List types
-                union_values = [
-                    get_args(u)[0] if get_origin(u) is list else u for u in union_values
-                ]
-                # Ignore injected NoneType when optional
-                concrete_union_values = [v for v in union_values if not is_subclass(v, type(None))]
-                dataclass_union_values = [v for v in concrete_union_values if is_dataclass(v)]
-                non_dataclass_union_values = [
-                    v for v in concrete_union_values if not is_dataclass(v)
-                ]
-                if not dataclass_union_values:
-                    continue
-                # Check if the key for this field already exists in the dictionary,
-                # if so it might map to one of these non dataclass fields and this
-                # can't be enforced
-                if non_dataclass_union_values and field.name in dd:
-                    continue
-                if len(dataclass_union_values) > 1:
-                    logger.warning(
-                        "more than one dataclass type possible for field {}, "
-                        "not extracting: {}".format(field.name, ", ".join(dataclass_union_values))
-                    )
-                    continue
-                f_type = dataclass_union_values[0]
-            origin = get_origin(f_type)
-            if origin:
-                f_type = origin
-            if is_subclass(f_type, EnhancedDataClassJsonMixin):
-                dd[field.name] = conform_dict(inner_d=dd, inner_config=f_type)
-        return dd
-
-    adjusted_dict = conform_dict(inner_d=flat_data, inner_config=config)
-    return config.from_dict(adjusted_dict, apply_name_overload=False)
+BaseModelT = TypeVar("BaseModelT", bound=BaseModel)
+
+
+def extract_config(flat_data: dict, config: Type[BaseModelT]) -> BaseModelT:
+    fields = config.model_fields
+    config.model_config = ConfigDict(extra="ignore")
+    field_names = [v.alias or k for k, v in fields.items()]
+    data = {k: v for k, v in flat_data.items() if k in field_names and v is not None}
+    if access_config := fields.get("access_config"):
+        access_config_type = access_config.annotation
+        # Check if raw type is wrapped by a secret
+        if (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Secret
+        ):
+            ac_subtypes = access_config_type.__args__
+            ac_fields = ac_subtypes[0].model_fields
+        elif issubclass(access_config_type, BaseModel):
+            ac_fields = access_config_type.model_fields
+        else:
+            raise TypeError(f"Unrecognized access_config type: {access_config_type}")
+        ac_field_names = [v.alias or k for k, v in ac_fields.items()]
+        data["access_config"] = {
+            k: v for k, v in flat_data.items() if k in ac_field_names and v is not None
+        }
+    return config.model_validate(obj=data)
 
 
 class Group(click.Group):
@@ -195,13 +144,11 @@ class Group(click.Group):
         This allows for subcommands to be called with the --help flag without breaking
         if parent command is missing any of its required parameters
         """
-
         try:
             return super().parse_args(ctx, args)
         except click.MissingParameter:
             if "--help" not in args:
                 raise
-
             # remove the required params so that help can display
             for param in self.params:
                 param.required = False

unstructured_ingest/v2/cli/utils/model_conversion.py

@@ -0,0 +1,199 @@
+import contextlib
+import datetime
+from collections import Counter
+from enum import EnumMeta
+from pathlib import Path
+from typing import Any, Callable, Literal, Optional, Type, TypedDict, Union, get_args, get_origin
+from uuid import UUID
+
+import click
+from annotated_types import Ge, Gt, Le, Lt, SupportsGe, SupportsGt, SupportsLe, SupportsLt
+from click import Option
+from pydantic import BaseModel, Secret, SecretStr
+from pydantic.fields import FieldInfo
+from pydantic.types import _SecretBase
+from pydantic_core import PydanticUndefined
+
+from unstructured_ingest.v2.cli.utils.click import DelimitedString, Dict
+
+NoneType = type(None)
+
+
+class _RangeDict(TypedDict, total=False):
+    """Represent arguments to `click.IntRange` or `click.FloatRange`."""
+
+    max: Union[SupportsLt, SupportsLe]
+    min: Union[SupportsGt, SupportsGe]
+    max_open: bool
+    min_open: bool
+
+
+def get_range_from_metadata(metadata: list[Any]) -> _RangeDict:
+    range_args: _RangeDict = {}
+    for constraint in metadata:
+        if isinstance(constraint, Le):
+            range_args["max"] = constraint.le
+            range_args["max_open"] = False
+        if isinstance(constraint, Lt):
+            range_args["max"] = constraint.lt
+            range_args["max_open"] = True
+        if isinstance(constraint, Ge):
+            range_args["min"] = constraint.ge
+            range_args["min_open"] = False
+        if isinstance(constraint, Gt):
+            range_args["min"] = constraint.gt
+            range_args["min_open"] = True
+    return range_args
+
+
+def is_boolean_flag(field_info: FieldInfo) -> bool:
+    annotation = field_info.annotation
+    raw_annotation = get_raw_type(annotation)
+    return raw_annotation is bool
+
+
+def get_raw_type(val: Any) -> Any:
+    field_args = get_args(val)
+    field_origin = get_origin(val)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return field_type
+    if val is SecretStr:
+        return str
+    return val
+
+
+def get_default_value_from_field(field: FieldInfo) -> Optional[Union[Any, Callable[[], Any]]]:
+    if field.default is not PydanticUndefined:
+        return field.default
+    elif field.default_factory is not None:
+        return field.default_factory
+    return None
+
+
+def get_option_name(field_name: str, field_info: FieldInfo) -> str:
+    field_name = field_info.alias or field_name
+    if field_name.startswith("--"):
+        field_name = field_name[2:]
+    field_name = field_name.lower().replace("_", "-")
+    if is_boolean_flag(field_info):
+        return f"--{field_name}/--no-{field_name}"
+    return f"--{field_name}"
+
+
+def get_numerical_type(field: FieldInfo) -> click.ParamType:
+    range_args = get_range_from_metadata(field.metadata)
+    if field.annotation is int:
+        if range_args:
+            return click.IntRange(**range_args)  # type: ignore[arg-type]
+        return click.INT
+    # Non-integer numerical types default to float
+    if range_args:
+        return click.FloatRange(**range_args)  # type: ignore[arg-type]
+    return click.FLOAT
+
+
+def get_type_from_annotation(field_type: Any) -> click.ParamType:
+    field_origin = get_origin(field_type)
+    field_args = get_args(field_type)
+    if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is Secret and len(field_args) == 1:
+        field_type = next(field_arg for field_arg in field_args if field_arg is not None)
+        return get_type_from_annotation(field_type=field_type)
+    if field_origin is list and len(field_args) == 1 and field_args[0] is str:
+        return DelimitedString()
+    if field_type is SecretStr:
+        return click.STRING
+    if dict in [field_type, field_origin]:
+        return Dict()
+    if field_type is str:
+        return click.STRING
+    if field_type is bool:
+        return click.BOOL
+    if field_type is UUID:
+        return click.UUID
+    if field_type is Path:
+        return click.Path(path_type=Path)
+    if field_type in (datetime.datetime, datetime.date):
+        return click.DateTime()
+    if field_origin is Literal:
+        return click.Choice(field_args)
+    if isinstance(field_type, EnumMeta):
+        values = [i.value for i in field_type]
+        return click.Choice(values)
+    raise TypeError(f"Unexpected field type: {field_type}")
+
+
+def _get_type_from_field(field: FieldInfo) -> click.ParamType:
+    raw_field_type = get_raw_type(field.annotation)
+
+    if raw_field_type in (int, float):
+        return get_numerical_type(field)
+    return get_type_from_annotation(field_type=field.annotation)
+
+
+def get_option_from_field(option_name: str, field_info: FieldInfo) -> Option:
+    param_decls = [option_name]
+    help = field_info.description or ""
+    if examples := field_info.examples:
+        help += f" [Examples: {', '.join(examples)}]"
+    option_kwargs = {
+        "type": _get_type_from_field(field_info),
+        "default": get_default_value_from_field(field_info),
+        "required": field_info.is_required(),
+        "help": help,
+        "is_flag": is_boolean_flag(field_info),
+        "show_default": field_info.default is not PydanticUndefined,
+    }
+    return click.Option(param_decls=param_decls, **option_kwargs)
+
+
+def is_subclass(x: Any, y: Any) -> bool:
+    with contextlib.suppress(TypeError):
+        return issubclass(x, y)
+
+    return False
+
+
+def post_check(options: list[Option]):
+    option_names = [option.name for option in options]
+    duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+    if duplicate_names:
+        raise ValueError(
+            "the following field name were reused, all must be unique: {}".format(
+                ", ".join(duplicate_names)
+            )
+        )
+
+
+def is_secret(value: Any) -> bool:
+    # Case Secret[int]
+    if hasattr(value, "__origin__") and hasattr(value, "__args__"):
+        origin = value.__origin__
+        return is_subclass(origin, _SecretBase)
+    # Case SecretStr
+    return is_subclass(value, _SecretBase)
+
+
+def options_from_base_model(model: Union[BaseModel, Type[BaseModel]]) -> list[Option]:
+    options = []
+    model_fields = model.model_fields
+    for field_name, field_info in model_fields.items():
+        if field_info.init is False:
+            continue
+        option_name = get_option_name(field_name=field_name, field_info=field_info)
+        raw_annotation = get_raw_type(field_info.annotation)
+        if is_subclass(raw_annotation, BaseModel):
+            options.extend(options_from_base_model(model=raw_annotation))
+        else:
+            if is_secret(field_info.annotation):
+                field_info.description = f"[sensitive] {field_info.description}"
+            options.append(get_option_from_field(option_name=option_name, field_info=field_info))
+
+    post_check(options=options)
+    return options

unstructured_ingest/v2/interfaces/__init__.py

@@ -1,6 +1,6 @@
 from .connector import AccessConfig, BaseConnector, ConnectionConfig
 from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
-from .file_data import FileData, SourceIdentifiers
+from .file_data import FileData, FileDataSourceMetadata, SourceIdentifiers
 from .indexer import Indexer, IndexerConfig
 from .process import BaseProcess
 from .processor import ProcessorConfig
@@ -26,4 +26,5 @@ __all__ = [
     "AccessConfig",
     "ConnectionConfig",
     "BaseConnector",
+    "FileDataSourceMetadata",
 ]

unstructured_ingest/v2/interfaces/connector.py

@@ -2,11 +2,10 @@ from abc import ABC
 from dataclasses import dataclass
 from typing import Any, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel, Secret
 
 
-@dataclass
-class AccessConfig(EnhancedDataClassJsonMixin):
+class AccessConfig(BaseModel):
     """Meant to designate holding any sensitive information associated with other configs
     and also for access specific configs."""
 
@@ -14,14 +13,13 @@ class AccessConfig(EnhancedDataClassJsonMixin):
 AccessConfigT = TypeVar("AccessConfigT", bound=AccessConfig)
 
 
-@dataclass
-class ConnectionConfig(EnhancedDataClassJsonMixin):
-    access_config: AccessConfigT
+class ConnectionConfig(BaseModel):
+    access_config: Secret[AccessConfigT]
 
     def get_access_config(self) -> dict[str, Any]:
         if not self.access_config:
             return {}
-        return self.access_config.to_dict(apply_name_overload=False)
+        return self.access_config.get_secret_value().dict()
 
 
 ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)

unstructured_ingest/v2/interfaces/downloader.py

@@ -1,18 +1,21 @@
 import os
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Optional, TypedDict, TypeVar, Union
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel, Field
+
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class DownloaderConfig(EnhancedDataClassJsonMixin):
-    download_dir: Optional[Path] = None
+class DownloaderConfig(BaseModel):
+    download_dir: Optional[Path] = Field(
+        default=None,
+        description="Where files are downloaded to, defaults to a location at"
+        "`$HOME/.cache/unstructured/ingest/<connector name>/<SHA256>`.",
+    )
 
 
 DownloaderConfigT = TypeVar("DownloaderConfigT", bound=DownloaderConfig)
@@ -30,6 +33,15 @@ class Downloader(BaseProcess, BaseConnector, ABC):
     connector_type: str
     download_config: DownloaderConfigT
 
+    def get_download_path(self, file_data: FileData) -> Optional[Path]:
+        if not file_data.source_identifiers:
+            return None
+        rel_path = file_data.source_identifiers.relative_path
+        if not rel_path:
+            return None
+        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
+        return self.download_dir / Path(rel_path)
+
     @staticmethod
     def is_float(value: str):
         try:
@@ -68,9 +80,6 @@ class Downloader(BaseProcess, BaseConnector, ABC):
     def is_async(self) -> bool:
         return True
 
-    def get_download_path(self, file_data: FileData) -> Optional[Path]:
-        return None
-
     @abstractmethod
     def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
         pass

unstructured_ingest/v2/interfaces/file_data.py

@@ -4,7 +4,6 @@ from pathlib import Path
 from typing import Any, Literal, Optional
 
 from dataclasses_json import DataClassJsonMixin
-from unstructured.documents.elements import DataSourceMetadata
 
 
 @dataclass
@@ -22,13 +21,25 @@ class SourceIdentifiers:
         return self.rel_path or self.fullpath
 
 
+@dataclass
+class FileDataSourceMetadata(DataClassJsonMixin):
+    url: Optional[str] = None
+    version: Optional[str] = None
+    record_locator: Optional[dict[str, Any]] = None
+    date_created: Optional[str] = None
+    date_modified: Optional[str] = None
+    date_processed: Optional[str] = None
+    permissions_data: Optional[list[dict[str, Any]]] = None
+    filesize_bytes: Optional[int] = None
+
+
 @dataclass
 class FileData(DataClassJsonMixin):
     identifier: str
     connector_type: str
     source_identifiers: Optional[SourceIdentifiers] = None
     doc_type: Literal["file", "batch"] = field(default="file")
-    metadata: DataSourceMetadata = field(default_factory=DataSourceMetadata)
+    metadata: FileDataSourceMetadata = field(default_factory=lambda: FileDataSourceMetadata())
     additional_metadata: dict[str, Any] = field(default_factory=dict)
     reprocess: bool = False
 

unstructured_ingest/v2/interfaces/indexer.py

@@ -1,15 +1,14 @@
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
 from typing import Any, Generator, Optional, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel
+
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class IndexerConfig(EnhancedDataClassJsonMixin):
+class IndexerConfig(BaseModel):
     pass
 
 

unstructured_ingest/v2/interfaces/process.py

@@ -8,13 +8,12 @@ class BaseProcess(ABC):
     def is_async(self) -> bool:
         return False
 
+    def precheck(self) -> None:
+        pass
+
     @abstractmethod
     def run(self, **kwargs: Any) -> Any:
         pass
 
     async def run_async(self, **kwargs: Any) -> Any:
         return self.run(**kwargs)
-
-    def check_connection(self):
-        # If the process requires external connections, run a quick check
-        pass

unstructured_ingest/v2/interfaces/processor.py

@@ -1,24 +1,24 @@
 import os
 from asyncio import Semaphore
-from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel, ConfigDict, Field
 
 DEFAULT_WORK_DIR = str((Path.home() / ".cache" / "unstructured" / "ingest" / "pipeline").resolve())
 
 
-@dataclass
-class ProcessorConfig(EnhancedDataClassJsonMixin):
+class ProcessorConfig(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     reprocess: bool = False
     verbose: bool = False
     tqdm: bool = False
-    work_dir: str = field(default_factory=lambda: DEFAULT_WORK_DIR)
+    work_dir: str = Field(default_factory=lambda: DEFAULT_WORK_DIR)
     num_processes: int = 2
     max_connections: Optional[int] = None
     raise_on_error: bool = False
-    disable_parallelism: bool = field(
+    disable_parallelism: bool = Field(
         default_factory=lambda: os.getenv("INGEST_DISABLE_PARALLELISM", "false").lower() == "true"
     )
     preserve_downloads: bool = False
@@ -28,10 +28,10 @@ class ProcessorConfig(EnhancedDataClassJsonMixin):
     uncompress: bool = False
 
     # Used to keep track of state in pipeline
-    status: dict = field(default_factory=dict)
-    semaphore: Optional[Semaphore] = field(init=False, default=None)
+    status: dict = Field(default_factory=dict)
+    semaphore: Optional[Semaphore] = Field(init=False, default=None)
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         if self.max_connections is not None:
             self.semaphore = Semaphore(self.max_connections)
 

unstructured_ingest/v2/interfaces/upload_stager.py

@@ -3,13 +3,13 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel
+
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class UploadStagerConfig(EnhancedDataClassJsonMixin):
+class UploadStagerConfig(BaseModel):
     pass
 
 

unstructured_ingest/v2/interfaces/uploader.py

@@ -3,14 +3,14 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, TypeVar
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin
+from pydantic import BaseModel
+
 from unstructured_ingest.v2.interfaces.connector import BaseConnector
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 
 
-@dataclass
-class UploaderConfig(EnhancedDataClassJsonMixin):
+class UploaderConfig(BaseModel):
     pass
 
 

unstructured_ingest/v2/pipeline/interfaces.py

@@ -92,7 +92,7 @@ class PipelineStep(ABC):
 
         if iterable:
             if len(iterable) == 1:
-                return [self.process_serially(iterable)]
+                return self.process_serially(iterable)
             if self.context.num_processes == 1:
                 return self.process_serially(iterable)
             with mp.Pool(
@@ -126,6 +126,8 @@ class PipelineStep(ABC):
             logger.info(
                 f"Calling {self.__class__.__name__} " f"with {len(iterable)} docs",  # type: ignore
             )
+        else:
+            logger.info(f"Calling {self.__class__.__name__} with no inputs")
         if self.context.async_supported and self.process.is_async():
             return self.process_async(iterable=iterable)
         if self.context.mp_supported:
@@ -146,8 +148,6 @@ class PipelineStep(ABC):
             logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
             if "file_data_path" in kwargs:
                 self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
-            else:
-                self.context.status[self.identifier] = {"step_error": str(e)}
             if self.context.raise_on_error:
                 raise e
             return None
@@ -160,8 +160,6 @@ class PipelineStep(ABC):
             logger.error(f"Exception raised while running {self.identifier}", exc_info=e)
             if "file_data_path" in kwargs:
                 self.context.status[kwargs["file_data_path"]] = {self.identifier: str(e)}
-            else:
-                self.context.status[self.identifier] = {"step_error": str(e)}
             if self.context.raise_on_error:
                 raise e
             return None