unstructured-ingest 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.7"  # pragma: no cover
+__version__ = "0.0.9"  # pragma: no cover

unstructured_ingest/v2/cli/utils/model_conversion.py

@@ -3,7 +3,18 @@ import datetime
 from collections import Counter
 from enum import EnumMeta
 from pathlib import Path
-from typing import Any, Callable, Literal, Optional, Type, TypedDict, Union, get_args, get_origin
+from typing import (
+    Annotated,
+    Any,
+    Callable,
+    Literal,
+    Optional,
+    Type,
+    TypedDict,
+    Union,
+    get_args,
+    get_origin,
+)
 from uuid import UUID
 
 import click
@@ -102,6 +113,11 @@ def get_type_from_annotation(field_type: Any) -> click.ParamType:
     if field_origin is Union and len(field_args) == 2 and NoneType in field_args:
         field_type = next(field_arg for field_arg in field_args if field_arg is not None)
         return get_type_from_annotation(field_type=field_type)
+    if field_origin is Annotated:
+        field_origin = field_args[0]
+        field_metadata = field_args[1]
+        if isinstance(field_metadata, click.ParamType):
+            return field_metadata
     if field_origin is Secret and len(field_args) == 1:
         field_type = next(field_arg for field_arg in field_args if field_arg is not None)
         return get_type_from_annotation(field_type=field_type)

unstructured_ingest/v2/interfaces/file_data.py

@@ -42,6 +42,7 @@ class FileData(DataClassJsonMixin):
     metadata: FileDataSourceMetadata = field(default_factory=lambda: FileDataSourceMetadata())
     additional_metadata: dict[str, Any] = field(default_factory=dict)
     reprocess: bool = False
+    local_download_path: Optional[str] = None
 
     @classmethod
     def from_file(cls, path: str) -> "FileData":

unstructured_ingest/v2/pipeline/steps/download.py

@@ -68,10 +68,9 @@ class DownloadStep(PipelineStep):
     def update_file_data(
         self, file_data: FileData, file_data_path: Path, download_path: Path
     ) -> None:
+        file_data.local_download_path = str(download_path.resolve())
         file_size_bytes = download_path.stat().st_size
-        changed = False
         if not file_data.metadata.filesize_bytes and file_size_bytes:
-            changed = True
             file_data.metadata.filesize_bytes = file_size_bytes
         if (
             file_data.metadata.filesize_bytes
@@ -82,12 +81,10 @@ class DownloadStep(PipelineStep):
                 f"({file_data.metadata.filesize_bytes}) doesn't "
                 f"match size of local file: {file_size_bytes}, updating"
             )
-            changed = True
             file_data.metadata.filesize_bytes = file_size_bytes
-        if changed:
-            logger.debug(f"Updating file data with new content: {file_data.to_dict()}")
-            with file_data_path.open("w") as file:
-                json.dump(file_data.to_dict(), file, indent=2)
+        logger.debug(f"Updating file data with new content: {file_data.to_dict()}")
+        with file_data_path.open("w") as file:
+            json.dump(file_data.to_dict(), file, indent=2)
 
     async def _run_async(self, fn: Callable, file_data_path: str) -> list[DownloadStepResponse]:
         file_data = FileData.from_file(path=file_data_path)

unstructured_ingest/v2/pipeline/steps/uncompress.py

@@ -1,4 +1,5 @@
 import asyncio
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, TypedDict
 
@@ -15,6 +16,7 @@ class UncompressStepResponse(TypedDict):
     path: str
 
 
+@dataclass
 class UncompressStep(PipelineStep):
     process: Uncompressor
     identifier: str = STEP_ID
@@ -23,21 +25,6 @@ class UncompressStep(PipelineStep):
         config = self.process.config.json() if self.process.config else None
         logger.info(f"Created {self.identifier} with configs: {config}")
 
-    def _run(self, path: str, file_data_path: str) -> list[UncompressStepResponse]:
-        file_data = FileData.from_file(path=file_data_path)
-        new_file_data = self.process.run(file_data=file_data)
-        responses = []
-        for new_file in new_file_data:
-            new_file_data_path = Path(file_data_path).parent / f"{new_file.identifier}.json"
-            new_file.to_file(path=str(new_file_data_path.resolve()))
-            responses.append(
-                UncompressStepResponse(
-                    path=new_file.source_identifiers.fullpath,
-                    file_data_path=str(new_file_data_path),
-                )
-            )
-        return responses
-
     async def _run_async(
         self, fn: Callable, path: str, file_data_path: str
     ) -> list[UncompressStepResponse]:
@@ -56,7 +43,7 @@ class UncompressStep(PipelineStep):
             new_file.to_file(path=str(new_file_data_path.resolve()))
             responses.append(
                 UncompressStepResponse(
-                    path=new_file.source_identifiers.fullpath,
+                    path=new_file.local_download_path,
                     file_data_path=str(new_file_data_path),
                 )
             )

unstructured_ingest/v2/processes/connectors/chroma.py

@@ -3,10 +3,11 @@ import uuid
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Annotated, Any, Optional
 
 from dateutil import parser
 from pydantic import Field, Secret
+from pydantic.functional_validators import BeforeValidator
 
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
@@ -21,9 +22,9 @@ from unstructured_ingest.v2.interfaces import (
     UploadStagerConfig,
 )
 from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
-    DestinationRegistryEntry,
-)
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+
+from .utils import conform_string_to_dict
 
 if TYPE_CHECKING:
     from chromadb import Client
@@ -32,10 +33,10 @@ CONNECTOR_TYPE = "chroma"
 
 
 class ChromaAccessConfig(AccessConfig):
-    settings: Optional[dict[str, str]] = Field(
+    settings: Optional[Annotated[dict, BeforeValidator(conform_string_to_dict)]] = Field(
         default=None, description="A dictionary of settings to communicate with the chroma server."
     )
-    headers: Optional[dict[str, str]] = Field(
+    headers: Optional[Annotated[dict, BeforeValidator(conform_string_to_dict)]] = Field(
         default=None, description="A dictionary of headers to send to the Chroma server."
     )
 

unstructured_ingest/v2/processes/connectors/google_drive.py

@@ -2,10 +2,11 @@ import io
 import json
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generator, Optional
+from typing import TYPE_CHECKING, Annotated, Any, Generator, Optional
 
 from dateutil import parser
 from pydantic import Field, Secret
+from pydantic.functional_validators import BeforeValidator
 
 from unstructured_ingest.error import (
     SourceConnectionError,
@@ -26,9 +27,9 @@ from unstructured_ingest.v2.interfaces import (
     download_responses,
 )
 from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.processes.connector_registry import (
-    SourceRegistryEntry,
-)
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
+
+from .utils import conform_string_to_dict
 
 CONNECTOR_TYPE = "google_drive"
 
@@ -38,7 +39,7 @@ if TYPE_CHECKING:
 
 
 class GoogleDriveAccessConfig(AccessConfig):
-    service_account_key: Optional[dict] = Field(
+    service_account_key: Optional[Annotated[dict, BeforeValidator(conform_string_to_dict)]] = Field(
         default=None, description="Credentials values to use for authentication"
     )
     service_account_key_path: Optional[Path] = Field(

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -67,7 +67,15 @@ class MilvusConnectionConfig(ConnectionConfig):
 
 
 class MilvusUploadStagerConfig(UploadStagerConfig):
-    pass
+
+    fields_to_include: Optional[list[str]] = None
+    """If set - list of fields to include in the output.
+    Unspecified fields are removed from the elements.
+    This action takse place after metadata flattening.
+    Missing fields will cause stager to throw KeyError."""
+
+    flatten_metadata: bool = True
+    """If set - flatten "metadata" key and put contents directly into data"""
 
 
 @dataclass
@@ -85,8 +93,26 @@ class MilvusUploadStager(UploadStager):
             pass
         return parser.parse(date_string).timestamp()
 
-    @classmethod
-    def conform_dict(cls, data: dict) -> None:
+    def conform_dict(self, data: dict) -> None:
+        if self.upload_stager_config.flatten_metadata and (metadata := data.pop("metadata", None)):
+            data.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
+
+        # TODO: milvus sdk doesn't seem to support defaults via the schema yet,
+        #  remove once that gets updated
+        defaults = {"is_continuation": False}
+        for default in defaults:
+            if default not in data:
+                data[default] = defaults[default]
+
+        if self.upload_stager_config.fields_to_include:
+            data_keys = set(data.keys())
+            for data_key in data_keys:
+                if data_key not in self.upload_stager_config.fields_to_include:
+                    data.pop(data_key)
+            for field_include_key in self.upload_stager_config.fields_to_include:
+                if field_include_key not in data:
+                    raise KeyError(f"Field '{field_include_key}' is missing in data!")
+
         datetime_columns = [
             "data_source_date_created",
             "data_source_date_modified",
@@ -96,21 +122,12 @@ class MilvusUploadStager(UploadStager):
 
         json_dumps_fields = ["languages", "data_source_permissions_data"]
 
-        # TODO: milvus sdk doesn't seem to support defaults via the schema yet,
-        #  remove once that gets updated
-        defaults = {"is_continuation": False}
-
-        if metadata := data.pop("metadata", None):
-            data.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
         for datetime_column in datetime_columns:
             if datetime_column in data:
-                data[datetime_column] = cls.parse_date_string(data[datetime_column])
+                data[datetime_column] = self.parse_date_string(data[datetime_column])
         for json_dumps_field in json_dumps_fields:
             if json_dumps_field in data:
                 data[json_dumps_field] = json.dumps(data[json_dumps_field])
-        for default in defaults:
-            if default not in data:
-                data[default] = defaults[default]
 
     def run(
         self,

unstructured_ingest/v2/processes/connectors/opensearch.py

@@ -39,22 +39,6 @@ heavily on the Elasticsearch connector code, inheriting the functionality as muc
 
 class OpenSearchAccessConfig(AccessConfig):
     password: Optional[str] = Field(default=None, description="password when using basic auth")
-    use_ssl: bool = Field(default=False, description="use ssl for the connection")
-    verify_certs: bool = Field(default=False, description="whether to verify SSL certificates")
-    ssl_show_warn: bool = Field(
-        default=False, description="show warning when verify certs is disabled"
-    )
-    ca_certs: Optional[Path] = Field(default=None, description="path to CA bundle")
-    client_cert: Optional[Path] = Field(
-        default=None,
-        description="path to the file containing the private key and the certificate,"
-        " or cert only if using client_key",
-    )
-    client_key: Optional[Path] = Field(
-        default=None,
-        description="path to the file containing the private key"
-        " if using separate cert and key files",
-    )
 
 
 class OpenSearchClientInput(BaseModel):
@@ -75,6 +59,23 @@ class OpenSearchConnectionConfig(ConnectionConfig):
         examples=["http://localhost:9200"],
     )
     username: Optional[str] = Field(default=None, description="username when using basic auth")
+    use_ssl: bool = Field(default=False, description="use ssl for the connection")
+    verify_certs: bool = Field(default=False, description="whether to verify SSL certificates")
+    ssl_show_warn: bool = Field(
+        default=False, description="show warning when verify certs is disabled"
+    )
+    ca_certs: Optional[Path] = Field(default=None, description="path to CA bundle")
+    client_cert: Optional[Path] = Field(
+        default=None,
+        description="path to the file containing the private key and the certificate,"
+        " or cert only if using client_key",
+    )
+    client_key: Optional[Path] = Field(
+        default=None,
+        description="path to the file containing the private key"
+        " if using separate cert and key files",
+    )
+
     access_config: Secret[OpenSearchAccessConfig]
 
     def get_client_kwargs(self) -> dict:
@@ -85,18 +86,18 @@ class OpenSearchConnectionConfig(ConnectionConfig):
         client_input_kwargs = {}
         if self.hosts:
             client_input_kwargs["hosts"] = self.hosts
-        if access_config.use_ssl:
-            client_input_kwargs["use_ssl"] = access_config.use_ssl
-        if access_config.verify_certs:
-            client_input_kwargs["verify_certs"] = access_config.verify_certs
-        if access_config.ssl_show_warn:
-            client_input_kwargs["ssl_show_warn"] = access_config.ssl_show_warn
-        if access_config.ca_certs:
-            client_input_kwargs["ca_certs"] = str(access_config.ca_certs)
-        if access_config.client_cert:
-            client_input_kwargs["client_cert"] = str(access_config.client_cert)
-        if access_config.client_key:
-            client_input_kwargs["client_key"] = str(access_config.client_key)
+        if self.use_ssl:
+            client_input_kwargs["use_ssl"] = self.use_ssl
+        if self.verify_certs:
+            client_input_kwargs["verify_certs"] = self.verify_certs
+        if self.ssl_show_warn:
+            client_input_kwargs["ssl_show_warn"] = self.ssl_show_warn
+        if self.ca_certs:
+            client_input_kwargs["ca_certs"] = str(self.ca_certs)
+        if self.client_cert:
+            client_input_kwargs["client_cert"] = str(self.client_cert)
+        if self.client_key:
+            client_input_kwargs["client_key"] = str(self.client_key)
         if self.username and access_config.password:
             client_input_kwargs["http_auth"] = (self.username, access_config.password)
         client_input = OpenSearchClientInput(**client_input_kwargs)

unstructured_ingest/v2/processes/connectors/utils.py

@@ -1,7 +1,9 @@
+import json
 from datetime import datetime
-from typing import Union
+from typing import Any, Union
 
 from dateutil import parser
+from pydantic import ValidationError
 
 
 def parse_datetime(date_value: Union[int, str, float, datetime]) -> datetime:
@@ -17,3 +19,11 @@ def parse_datetime(date_value: Union[int, str, float, datetime]) -> datetime:
         return datetime.fromtimestamp(timestamp)
     except ValueError:
         return parser.parse(date_value)
+
+
+def conform_string_to_dict(value: Any) -> dict:
+    if isinstance(value, dict):
+        return value
+    if isinstance(value, str):
+        return json.loads(value)
+    raise ValidationError(f"Input could not be mapped to a valid dict: {value}")

unstructured_ingest/v2/processes/uncompress.py

@@ -3,12 +3,14 @@ from copy import copy
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any
+from uuid import NAMESPACE_DNS, uuid5
 
 from pydantic import BaseModel
 
 from unstructured_ingest.utils.compression import TAR_FILE_EXT, ZIP_FILE_EXT, uncompress_file
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.interfaces.process import BaseProcess
+from unstructured_ingest.v2.logger import logger
 
 
 class UncompressConfig(BaseModel):
@@ -23,19 +25,35 @@ class Uncompressor(BaseProcess, ABC):
         return True
 
     def run(self, file_data: FileData, **kwargs: Any) -> list[FileData]:
-        local_filepath = Path(file_data.source_identifiers.fullpath)
+        local_filepath = Path(file_data.local_download_path)
         if local_filepath.suffix not in TAR_FILE_EXT + ZIP_FILE_EXT:
             return [file_data]
         new_path = uncompress_file(filename=str(local_filepath))
         new_files = [i for i in Path(new_path).rglob("*") if i.is_file()]
         responses = []
+        logger.debug(
+            "uncompressed {} files from original file {}: {}".format(
+                len(new_files), local_filepath, ", ".join([str(f) for f in new_files])
+            )
+        )
         for f in new_files:
             new_file_data = copy(file_data)
-            new_file_data.source_identifiers.fullpath = str(f)
-            if new_file_data.source_identifiers.rel_path:
-                new_file_data.source_identifiers.rel_path = str(f).replace(
-                    str(local_filepath.parent), ""
-                )[1:]
+            new_file_data.identifier = str(uuid5(NAMESPACE_DNS, str(f)))
+            new_file_data.local_download_path = str(f.resolve())
+            new_rel_download_path = str(f).replace(str(Path(local_filepath.parent)), "")[1:]
+            new_file_data.source_identifiers = SourceIdentifiers(
+                filename=f.name,
+                fullpath=file_data.source_identifiers.fullpath.replace(
+                    file_data.source_identifiers.filename, new_rel_download_path
+                ),
+                rel_path=(
+                    file_data.source_identifiers.rel_path.replace(
+                        file_data.source_identifiers.filename, new_rel_download_path
+                    )
+                    if file_data.source_identifiers.rel_path
+                    else None
+                ),
+            )
             responses.append(new_file_data)
         return responses