unstructured-ingest 0.0.19__py3-none-any.whl → 0.0.22__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -22,7 +23,6 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecUploader,
     FsspecUploaderConfig,
 )
-from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_dict
 
 CONNECTOR_TYPE = "dropbox"
 
@@ -49,6 +49,40 @@ class DropboxIndexer(FsspecIndexer):
     index_config: DropboxIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"].lstrip("/")
+        date_created = None
+        date_modified = None
+        server_modified = file_data.get("server_modified")
+        client_modified = file_data.get("client_modified")
+        if server_modified and client_modified and server_modified > client_modified:
+            date_created = str(client_modified.timestamp())
+            date_modified = str(server_modified.timestamp())
+        elif server_modified and client_modified and server_modified < client_modified:
+            date_created = str(server_modified.timestamp())
+            date_modified = str(client_modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("content_hash")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def __post_init__(self):
         # dropbox expects the path to start with a /
@@ -63,12 +97,6 @@ class DropboxIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
-    def sterilize_info(self, path) -> dict:
-        # the fs.info method defined in the dropboxdrivefs library expects a "url"
-        # kwarg rather than "path"; though both refer to the same thing
-        info = self.fs.info(url=path)
-        return sterilize_dict(data=info)
-
 
 class DropboxDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -1,10 +1,7 @@
 from __future__ import annotations
 
-import contextlib
 from dataclasses import dataclass, field
-from datetime import datetime
 from pathlib import Path
-from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
@@ -113,18 +110,13 @@ class FsspecIndexer(Indexer):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
-    def list_files(self) -> list[str]:
+    def get_file_data(self) -> list[dict[str, Any]]:
         if not self.index_config.recursive:
             # fs.ls does not walk directories
             # directories that are listed in cloud storage can cause problems
             # because they are seen as 0 byte files
-            found = self.fs.ls(self.index_config.path_without_protocol, detail=True)
-            if isinstance(found, list):
-                return [
-                    x.get("name") for x in found if x.get("size") > 0 and x.get("type") == "file"
-                ]
-            else:
-                raise TypeError(f"unhandled response type from ls: {type(found)}")
+            files = self.fs.ls(self.index_config.path_without_protocol, detail=True)
+
         else:
             # fs.find will recursively walk directories
             # "size" is a common key for all the cloud protocols with fs
@@ -132,84 +124,40 @@ class FsspecIndexer(Indexer):
                 self.index_config.path_without_protocol,
                 detail=True,
             )
-            if isinstance(found, dict):
-                return [
-                    k for k, v in found.items() if v.get("size") > 0 and v.get("type") == "file"
-                ]
-            else:
-                raise TypeError(f"unhandled response type from find: {type(found)}")
-
-    def get_metadata(self, path: str) -> FileDataSourceMetadata:
-        date_created = None
-        date_modified = None
-        file_size = None
-        try:
-            created: Optional[Any] = self.fs.created(path)
-            if created:
-                if isinstance(created, datetime):
-                    date_created = str(created.timestamp())
-                else:
-                    date_created = str(created)
-        except NotImplementedError:
-            pass
+            files = found.values()
+        filtered_files = [
+            file for file in files if file.get("size") > 0 and file.get("type") == "file"
+        ]
+        return filtered_files
 
-        try:
-            modified: Optional[Any] = self.fs.modified(path)
-            if modified:
-                if isinstance(modified, datetime):
-                    date_modified = str(modified.timestamp())
-                else:
-                    date_modified = str(modified)
-        except NotImplementedError:
-            pass
-        with contextlib.suppress(AttributeError):
-            file_size = self.fs.size(path)
-
-        version = self.fs.checksum(path)
-        metadata: dict[str, str] = {}
-        with contextlib.suppress(AttributeError):
-            metadata = self.fs.metadata(path)
-        record_locator = {
-            "protocol": self.index_config.protocol,
-            "remote_file_path": self.index_config.remote_url,
-        }
-        file_stat = self.fs.stat(path=path)
-        if file_id := file_stat.get("id"):
-            record_locator["file_id"] = file_id
-        if metadata:
-            record_locator["metadata"] = metadata
-        return FileDataSourceMetadata(
-            date_created=date_created,
-            date_modified=date_modified,
-            date_processed=str(time()),
-            version=str(version),
-            url=f"{self.index_config.protocol}://{path}",
-            record_locator=record_locator,
-            filesize_bytes=file_size,
-        )
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        raise NotImplementedError()
+
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
 
-    def sterilize_info(self, path) -> dict:
-        info = self.fs.info(path=path)
-        return sterilize_dict(data=info)
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data)
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        files = self.list_files()
-        for file in files:
+        files = self.get_file_data()
+        for file_data in files:
+            file_path = self.get_path(file_data=file_data)
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
-            rel_path = file.replace(self.index_config.path_without_protocol, "").lstrip("/")
+            rel_path = file_path.replace(self.index_config.path_without_protocol, "").lstrip("/")
 
-            additional_metadata = self.sterilize_info(path=file)
-            additional_metadata["original_file_path"] = file
+            additional_metadata = self.sterilize_info(file_data=file_data)
+            additional_metadata["original_file_path"] = file_path
             yield FileData(
-                identifier=str(uuid5(NAMESPACE_DNS, file)),
+                identifier=str(uuid5(NAMESPACE_DNS, file_path)),
                 connector_type=self.connector_type,
                 source_identifiers=SourceIdentifiers(
-                    filename=Path(file).name,
+                    filename=Path(file_path).name,
                     rel_path=rel_path or None,
-                    fullpath=file,
+                    fullpath=file_path,
                 ),
-                metadata=self.get_metadata(path=file),
+                metadata=self.get_metadata(file_data=file_data),
                 additional_metadata=additional_metadata,
             )
 

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -2,13 +2,15 @@ from __future__ import annotations
 
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional, Union
 
+from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import json_to_dict
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -106,6 +108,33 @@ class GcsIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = None
+        date_modified = None
+        if modified_at_str := file_data.get("updated"):
+            date_modified = parser.parse(modified_at_str).timestamp()
+        if created_at_str := file_data.get("timeCreated"):
+            date_created = parser.parse(created_at_str).timestamp()
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("etag")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class GcsDownloaderConfig(FsspecDownloaderConfig):
     pass

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -1,6 +1,5 @@
 import contextlib
 from dataclasses import dataclass, field
-from datetime import datetime
 from pathlib import Path
 from time import time
 from typing import Any, Generator, Optional
@@ -69,7 +68,7 @@ class S3ConnectionConfig(FsspecConnectionConfig):
 
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs.update(
-            {k: v for k, v in self.access_config.get_secret_value().dict().items() if v}
+            {k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v}
         )
         return access_configs
 
@@ -80,27 +79,25 @@ class S3Indexer(FsspecIndexer):
     index_config: S3IndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
-    def get_metadata(self, path: str) -> FileDataSourceMetadata:
+    def get_path(self, file_data: dict) -> str:
+        return file_data["Key"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["Key"]
         date_created = None
         date_modified = None
-        file_size = None
-        try:
-            modified: Optional[datetime] = self.fs.modified(path)
-            if modified:
-                date_created = str(modified.timestamp())
-                date_modified = str(modified.timestamp())
-        except NotImplementedError:
-            pass
-        with contextlib.suppress(AttributeError):
-            file_size = self.fs.size(path)
+        modified = file_data.get("LastModified")
+        if modified:
+            date_created = str(modified.timestamp())
+            date_modified = str(modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+        file_size = file_size or file_data.get("Size")
 
-        version = None
-        info: dict[str, Any] = self.fs.info(path)
-        if etag := info.get("ETag"):
-            version = str(etag).rstrip('"').lstrip('"')
+        version = file_data.get("ETag").rstrip('"').lstrip('"') if "ETag" in file_data else None
         metadata: dict[str, str] = {}
         with contextlib.suppress(AttributeError):
-            metadata = self.fs.metadata(path)
+            metadata = self.fs.metadata(path=path)
         record_locator = {
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
+from time import time
 from typing import Any, Generator, Optional
 from urllib.parse import urlparse
 
 from pydantic import Field, Secret
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
-from unstructured_ingest.v2.interfaces import DownloadResponse, FileData
+from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
@@ -96,6 +97,26 @@ class SftpIndexer(FsspecIndexer):
     def precheck(self) -> None:
         super().precheck()
 
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = file_data.get("time").timestamp() if "time" in file_data else None
+        date_modified = file_data.get("mtime").timestamp() if "mtime" in file_data else None
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
 
 class SftpDownloaderConfig(FsspecDownloaderConfig):
     remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -48,8 +48,8 @@ class MilvusConnectionConfig(ConnectionConfig):
 
     def get_connection_kwargs(self) -> dict[str, Any]:
         access_config = self.access_config.get_secret_value()
-        access_config_dict = access_config.dict()
-        connection_config_dict = self.dict()
+        access_config_dict = access_config.model_dump()
+        connection_config_dict = self.model_dump()
         connection_config_dict.pop("access_config", None)
         connection_config_dict.update(access_config_dict)
         # Drop any that were not set explicitly

unstructured_ingest/v2/processes/connectors/opensearch.py

@@ -101,8 +101,8 @@ class OpenSearchConnectionConfig(ConnectionConfig):
         if self.username and access_config.password:
             client_input_kwargs["http_auth"] = (self.username, access_config.password)
         client_input = OpenSearchClientInput(**client_input_kwargs)
-        logger.debug(f"opensearch client inputs mapped to: {client_input.dict()}")
-        client_kwargs = client_input.dict()
+        logger.debug(f"opensearch client inputs mapped to: {client_input.model_dump()}")
+        client_kwargs = client_input.model_dump()
         if client_input.http_auth is not None:
             client_kwargs["http_auth"] = client_input.http_auth.get_secret_value()
         client_kwargs = {k: v for k, v in client_kwargs.items() if v is not None}

unstructured_ingest/v2/processes/partitioner.py

@@ -1,8 +1,7 @@
-import asyncio
 from abc import ABC
-from dataclasses import dataclass, fields
+from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import Any, Optional
 
 from pydantic import BaseModel, Field, SecretStr
 
@@ -10,11 +9,7 @@ from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
-
-if TYPE_CHECKING:
-    from unstructured_client import UnstructuredClient
-    from unstructured_client.models.operations import PartitionRequest
-    from unstructured_client.models.shared import PartitionParameters
+from unstructured_ingest.v2.unstructured_api import call_api
 
 
 class PartitionerConfig(BaseModel):
@@ -154,60 +149,19 @@ class Partitioner(BaseProcess, ABC):
         )
         return self.postprocess(elements=elements_to_dicts(elements))
 
-    async def call_api(self, client: "UnstructuredClient", request: "PartitionRequest"):
-        # TODO when client supports async, run without using run_in_executor
-        # isolate the IO heavy call
-        loop = asyncio.get_event_loop()
-        return await loop.run_in_executor(None, client.general.partition, request)
-
-    def create_partition_parameters(self, filename: Path) -> "PartitionParameters":
-        from unstructured_client.models.shared import Files, PartitionParameters
-
-        partition_request = self.config.to_partition_kwargs()
-
-        # NOTE(austin): PartitionParameters is a Pydantic model in v0.26.0
-        # Prior to this it was a dataclass which doesn't have .__fields
-        try:
-            possible_fields = PartitionParameters.__fields__
-        except AttributeError:
-            possible_fields = [f.name for f in fields(PartitionParameters)]
-
-        filtered_partition_request = {
-            k: v for k, v in partition_request.items() if k in possible_fields
-        }
-        if len(filtered_partition_request) != len(partition_request):
-            logger.debug(
-                "Following fields were omitted due to not being "
-                "supported by the currently used unstructured client: {}".format(
-                    ", ".join([v for v in partition_request if v not in filtered_partition_request])
-                )
-            )
-        logger.debug(f"using hosted partitioner with kwargs: {partition_request}")
-        with open(filename, "rb") as f:
-            files = Files(
-                content=f.read(),
-                file_name=str(filename.resolve()),
-            )
-            filtered_partition_request["files"] = files
-        partition_params = PartitionParameters(**filtered_partition_request)
-        return partition_params
-
     @requires_dependencies(dependencies=["unstructured_client"], extras="remote")
     async def partition_via_api(
         self, filename: Path, metadata: Optional[dict] = None, **kwargs
     ) -> list[dict]:
-        from unstructured_client import UnstructuredClient
-        from unstructured_client.models.operations import PartitionRequest
-
         logger.debug(f"partitioning file {filename} with metadata: {metadata}")
-        client = UnstructuredClient(
+
+        elements = await call_api(
             server_url=self.config.partition_endpoint,
-            api_key_auth=self.config.api_key.get_secret_value(),
+            api_key=self.config.api_key.get_secret_value(),
+            filename=filename,
+            api_parameters=self.config.to_partition_kwargs(),
         )
-        partition_params = self.create_partition_parameters(filename=filename)
-        partition_request = PartitionRequest(partition_params)
-        resp = await self.call_api(client=client, request=partition_request)
-        elements = resp.elements or []
+
         # Append the data source metadata the auto partition does for you
         for element in elements:
             element["metadata"]["data_source"] = metadata

unstructured_ingest/v2/unstructured_api.py

@@ -0,0 +1,87 @@
+import asyncio
+from dataclasses import fields
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+from unstructured_ingest.v2.logger import logger
+
+if TYPE_CHECKING:
+    from unstructured_client.models.operations import PartitionRequest
+
+
+def create_partition_request(filename: Path, parameters_dict: dict) -> "PartitionRequest":
+    """Given a filename and a dict of API parameters, return a PartitionRequest for use
+    by unstructured-client. Remove any params that aren't recognized by the SDK.
+
+    Args:
+        filename: Path to the file being partitioned
+        parameters_dict: A mapping of all API params we want to send
+
+    Returns: A PartitionRequest containing the file and all valid params
+    """
+    from unstructured_client.models.operations import PartitionRequest
+    from unstructured_client.models.shared import Files, PartitionParameters
+
+    # NOTE(austin): PartitionParameters is a Pydantic model in v0.26.0
+    # Prior to this it was a dataclass which doesn't have .__fields
+    try:
+        possible_fields = PartitionParameters.__fields__
+    except AttributeError:
+        possible_fields = [f.name for f in fields(PartitionParameters)]
+
+    filtered_partition_request = {k: v for k, v in parameters_dict.items() if k in possible_fields}
+    if len(filtered_partition_request) != len(parameters_dict):
+        logger.debug(
+            "Following fields were omitted due to not being "
+            "supported by the currently used unstructured client: {}".format(
+                ", ".join([v for v in parameters_dict if v not in filtered_partition_request])
+            )
+        )
+
+    logger.debug(f"using hosted partitioner with kwargs: {parameters_dict}")
+
+    with open(filename, "rb") as f:
+        files = Files(
+            content=f.read(),
+            file_name=str(filename.resolve()),
+        )
+        filtered_partition_request["files"] = files
+
+    partition_params = PartitionParameters(**filtered_partition_request)
+
+    return PartitionRequest(partition_parameters=partition_params)
+
+
+async def call_api(
+    server_url: Optional[str], api_key: Optional[str], filename: Path, api_parameters: dict
+) -> list[dict]:
+    """Call the Unstructured API using unstructured-client.
+
+    Args:
+        server_url: The base URL where the API is hosted
+        api_key: The user's API key (can be empty if this is a self hosted API)
+        filename: Path to the file being partitioned
+        api_parameters: A dict containing the requested API parameters
+
+    Returns: A list of the file's elements, or an empty list if there was an error
+    """
+    from unstructured_client import UnstructuredClient
+
+    client = UnstructuredClient(
+        server_url=server_url,
+        api_key_auth=api_key,
+    )
+    partition_request = create_partition_request(filename=filename, parameters_dict=api_parameters)
+
+    # TODO when client supports async, run without using run_in_executor
+    # isolate the IO heavy call
+    loop = asyncio.get_event_loop()
+
+    # Note(austin) - The partition calls needs request to be a keyword arg
+    # We have to use partial to do this, we can't pass request=request into run_in_executor
+    partition_call = partial(client.general.partition, request=partition_request)
+
+    res = await loop.run_in_executor(None, partition_call)
+
+    return res.elements or []

unstructured_ingest/v2/utils.py

@@ -19,7 +19,7 @@ def is_secret(value: Any) -> bool:
 
 def serialize_base_model(model: BaseModel) -> dict:
     # To get the full serialized dict regardless of if values are marked as Secret
-    model_dict = model.dict()
+    model_dict = model.model_dump()
     return serialize_base_dict(model_dict=model_dict)