unstructured-ingest 1.0.56__tar.gz → 1.1.2__tar.gz

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.56
+Version: 1.1.2
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: <3.13,>=3.9
+Requires-Dist: certifi>=2025.7.14
 Requires-Dist: click
 Requires-Dist: opentelemetry-sdk
 Requires-Dist: pydantic>=2.7

unstructured_ingest-1.1.2/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.1.2"  # pragma: no cover

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/embed/azure_openai.py

@@ -9,6 +9,7 @@ from unstructured_ingest.embed.openai import (
     OpenAIEmbeddingEncoder,
 )
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.tls import ssl_context_with_optional_ca_override
 
 if TYPE_CHECKING:
     from openai import AsyncAzureOpenAI, AzureOpenAI
@@ -23,9 +24,11 @@ class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
 
     @requires_dependencies(["openai"], extras="openai")
     def get_client(self) -> "AzureOpenAI":
-        from openai import AzureOpenAI
+        from openai import AzureOpenAI, DefaultHttpxClient
 
+        client = DefaultHttpxClient(verify=ssl_context_with_optional_ca_override())
         return AzureOpenAI(
+            http_client=client,
             api_key=self.api_key.get_secret_value(),
             api_version=self.api_version,
             azure_endpoint=self.azure_endpoint,
@@ -33,9 +36,11 @@ class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
 
     @requires_dependencies(["openai"], extras="openai")
     def get_async_client(self) -> "AsyncAzureOpenAI":
-        from openai import AsyncAzureOpenAI
+        from openai import AsyncAzureOpenAI, DefaultAsyncHttpxClient
 
+        client = DefaultAsyncHttpxClient(verify=ssl_context_with_optional_ca_override())
         return AsyncAzureOpenAI(
+            http_client=client,
             api_key=self.api_key.get_secret_value(),
             api_version=self.api_version,
             azure_endpoint=self.azure_endpoint,

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/embed/openai.py

@@ -18,6 +18,7 @@ from unstructured_ingest.errors_v2 import (
 )
 from unstructured_ingest.logger import logger
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.tls import ssl_context_with_optional_ca_override
 
 if TYPE_CHECKING:
     from openai import AsyncOpenAI, OpenAI
@@ -86,15 +87,21 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
 
     @requires_dependencies(["openai"], extras="openai")
     def get_client(self) -> "OpenAI":
-        from openai import OpenAI
+        from openai import DefaultHttpxClient, OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        client = DefaultHttpxClient(verify=ssl_context_with_optional_ca_override())
+        return OpenAI(
+            api_key=self.api_key.get_secret_value(), http_client=client, base_url=self.base_url
+        )
 
     @requires_dependencies(["openai"], extras="openai")
     def get_async_client(self) -> "AsyncOpenAI":
-        from openai import AsyncOpenAI
+        from openai import AsyncOpenAI, DefaultAsyncHttpxClient
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        client = DefaultAsyncHttpxClient(verify=ssl_context_with_optional_ca_override())
+        return AsyncOpenAI(
+            api_key=self.api_key.get_secret_value(), http_client=client, base_url=self.base_url
+        )
 
 
 @dataclass

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/interfaces/connector.py

@@ -5,6 +5,8 @@ from typing import Any, TypeVar, Union
 from pydantic import BaseModel, Secret, model_validator
 from pydantic.types import _SecretBase
 
+from unstructured_ingest.processes.utils.logging.connector import ConnectorLoggingMixin
+
 
 class AccessConfig(BaseModel):
     """Meant to designate holding any sensitive information associated with other configs
@@ -46,5 +48,9 @@ ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)
 
 
 @dataclass
-class BaseConnector(ABC):
+class BaseConnector(ABC, ConnectorLoggingMixin):
     connection_config: ConnectionConfigT
+
+    def __post_init__(self):
+        """Initialize the logging mixin after dataclass initialization."""
+        ConnectorLoggingMixin.__init__(self)

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/otel.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from dataclasses import dataclass, field
 from typing import Callable, ClassVar, Optional, Protocol, Sequence
@@ -31,13 +32,27 @@ class LogSpanExporter(ConsoleSpanExporter):
             self.log_out(self.formatter(span))
         return SpanExportResult.SUCCESS
 
+def get_log_out() -> Callable:
+    level_names_mapping = {
+        'CRITICAL': logging.CRITICAL,
+        'FATAL': logging.FATAL,
+        'ERROR': logging.ERROR,
+        'WARN': logging.WARNING,
+        'WARNING': logging.WARNING,
+        'INFO': logging.INFO,
+        'DEBUG': logging.DEBUG,
+        'NOTSET': logging.NOTSET,
+    }
+    log_level = os.getenv("OTEL_LOG_LEVEL", "DEBUG").upper()
+    log_level_int = level_names_mapping.get(log_level, logging.DEBUG)
+    return lambda message: logger.log(log_level_int, message)
 
 @dataclass
 class OtelHandler:
     otel_endpoint: Optional[str] = None
     service_name: str = "unstructured-ingest"
     trace_provider: TracerProvider = field(init=False)
-    log_out: Callable = field(default=logger.info)
+    log_out: Callable = field(default=get_log_out())
     trace_context_key: ClassVar[str] = "_trace_context"
 
     def init_trace(self):

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/processes/connectors/delta_table.py

@@ -48,7 +48,8 @@ class DeltaTableAccessConfig(AccessConfig):
 
 class DeltaTableConnectionConfig(ConnectionConfig):
     access_config: Secret[DeltaTableAccessConfig] = Field(
-        default=Secret(DeltaTableAccessConfig()), validate_default=True
+        default_factory=lambda: Secret[DeltaTableAccessConfig](DeltaTableAccessConfig()),
+        validate_default=True,
     )
     aws_region: Optional[str] = Field(default=None, description="AWS Region")
     table_uri: str = Field(

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -28,7 +28,6 @@ from unstructured_ingest.interfaces import (
     Uploader,
     UploaderConfig,
 )
-from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connectors.fsspec.utils import sterilize_dict
 from unstructured_ingest.utils.filesystem import mkdir_concurrent_safe
 
@@ -106,6 +105,12 @@ class FsspecIndexer(Indexer):
     def precheck(self) -> None:
         from fsspec import get_filesystem_class
 
+        self.log_operation_start(
+            "Connection validation",
+            protocol=self.index_config.protocol,
+            path=self.index_config.path_without_protocol,
+        )
+
         try:
             fs = get_filesystem_class(self.index_config.protocol)(
                 **self.connection_config.get_access_config(),
@@ -113,13 +118,24 @@ class FsspecIndexer(Indexer):
             files = fs.ls(path=self.index_config.path_without_protocol, detail=True)
             valid_files = [x.get("name") for x in files if x.get("type") == "file"]
             if not valid_files:
+                self.log_operation_complete("Connection validation", count=0)
                 return
             file_to_sample = valid_files[0]
-            logger.debug(f"attempting to make HEAD request for file: {file_to_sample}")
+            self.log_debug(f"attempting to make HEAD request for file: {file_to_sample}")
             with self.connection_config.get_client(protocol=self.index_config.protocol) as client:
                 client.head(path=file_to_sample)
+
+            self.log_connection_validated(
+                connector_type=self.connector_type,
+                endpoint=f"{self.index_config.protocol}://{self.index_config.path_without_protocol}",
+            )
+
         except Exception as e:
-            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            self.log_connection_failed(
+                connector_type=self.connector_type,
+                error=e,
+                endpoint=f"{self.index_config.protocol}://{self.index_config.path_without_protocol}",
+            )
             raise self.wrap_error(e=e)
 
     def get_file_info(self) -> list[dict[str, Any]]:
@@ -150,7 +166,7 @@ class FsspecIndexer(Indexer):
 
     def sample_n_files(self, files: list[dict[str, Any]], n) -> list[dict[str, Any]]:
         if len(files) <= n:
-            logger.warning(
+            self.log_warning(
                 f"number of files to be sampled={n} is not smaller than the number"
                 f" of files found ({len(files)}). Returning all of the files as the"
                 " sample."
@@ -201,9 +217,22 @@ class FsspecIndexer(Indexer):
         init_file_data.additional_metadata = self.get_metadata(file_info=file_info)
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        self.log_indexing_start(f"{self.connector_type} files")
+
         files = self.get_file_info()
-        for file_info in files:
+        total_files = len(files)
+
+        self.log_operation_start("File indexing", total_files=total_files)
+
+        for i, file_info in enumerate(files):
             file_path = self.get_path(file_info=file_info)
+
+            # Only log progress for larger operations
+            if total_files > 5:
+                self.log_progress(
+                    current=i + 1, total=total_files, item_type="files", operation="Indexing"
+                )
+
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
             rel_path = file_path.replace(self.index_config.path_without_protocol, "").lstrip("/")
@@ -223,6 +252,8 @@ class FsspecIndexer(Indexer):
                 display_name=file_path,
             )
 
+        self.log_indexing_complete(f"{self.connector_type} files", total_files)
+
 
 class FsspecDownloaderConfig(DownloaderConfig):
     pass
@@ -272,25 +303,57 @@ class FsspecDownloader(Downloader):
     def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
         mkdir_concurrent_safe(download_path.parent)
+
+        rpath = file_data.additional_metadata["original_file_path"]
+        file_size = file_data.metadata.filesize_bytes
+        self.log_download_start(file_path=rpath, file_id=file_data.identifier, file_size=file_size)
+
         try:
-            rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:
                 client.get_file(rpath=rpath, lpath=download_path.as_posix())
             self.handle_directory_download(lpath=download_path)
+
         except Exception as e:
+            self.log_error(
+                "File download failed",
+                error=e,
+                context={"file_path": rpath, "file_id": file_data.identifier},
+            )
             raise self.wrap_error(e=e)
+
+        self.log_download_complete(
+            file_path=rpath,
+            file_id=file_data.identifier,
+            download_path=str(download_path),
+        )
+
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
     async def async_run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
         mkdir_concurrent_safe(download_path.parent)
+        rpath = file_data.additional_metadata["original_file_path"]
+        file_size = file_data.metadata.filesize_bytes
+        self.log_download_start(file_path=rpath, file_id=file_data.identifier, file_size=file_size)
+
         try:
-            rpath = file_data.additional_metadata["original_file_path"]
             with self.connection_config.get_client(protocol=self.protocol) as client:
                 await client.get_file(rpath=rpath, lpath=download_path.as_posix())
             self.handle_directory_download(lpath=download_path)
         except Exception as e:
+            self.log_error(
+                "File download failed",
+                error=e,
+                context={"file_path": rpath, "file_id": file_data.identifier},
+            )
             raise self.wrap_error(e=e)
+
+        self.log_download_complete(
+            file_path=rpath,
+            file_id=file_data.identifier,
+            download_path=str(download_path),
+        )
+
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
 
@@ -321,6 +384,7 @@ class FsspecUploader(Uploader):
         )
 
     def __post_init__(self):
+        super().__post_init__()
         # TODO once python3.9 no longer supported and kw_only is allowed in dataclasses, remove:
         if not self.upload_config:
             raise TypeError(
@@ -334,6 +398,8 @@ class FsspecUploader(Uploader):
     def precheck(self) -> None:
         from fsspec import get_filesystem_class
 
+        self.log_operation_start("Connection validation", protocol=self.upload_config.protocol)
+
         try:
             fs = get_filesystem_class(self.upload_config.protocol)(
                 **self.connection_config.get_access_config(),
@@ -341,7 +407,16 @@ class FsspecUploader(Uploader):
             upload_path = Path(self.upload_config.path_without_protocol) / "_empty"
             fs.write_bytes(path=upload_path.as_posix(), value=b"")
         except Exception as e:
+            self.log_connection_failed(
+                connector_type=self.connector_type,
+                error=e,
+                endpoint=f"{self.upload_config.protocol}://{self.upload_config.path_without_protocol}",
+            )
             raise self.wrap_error(e=e)
+        self.log_connection_validated(
+            connector_type=self.connector_type,
+            endpoint=f"{self.upload_config.protocol}://{self.upload_config.path_without_protocol}",
+        )
 
     def get_upload_path(self, file_data: FileData) -> Path:
         upload_path = Path(
@@ -353,14 +428,31 @@ class FsspecUploader(Uploader):
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         path_str = str(path.resolve())
         upload_path = self.get_upload_path(file_data=file_data)
-        logger.debug(f"writing local file {path_str} to {upload_path}")
-        with self.connection_config.get_client(protocol=self.upload_config.protocol) as client:
-            client.upload(lpath=path_str, rpath=upload_path.as_posix())
+        self.log_upload_start(file_path=path_str, destination=upload_path.as_posix())
+        try:
+            with self.connection_config.get_client(protocol=self.upload_config.protocol) as client:
+                client.upload(lpath=path_str, rpath=upload_path.as_posix())
+        except Exception as e:
+            self.log_error(
+                "File upload failed",
+                error=e,
+                context={"file_path": path_str, "destination": upload_path.as_posix()},
+            )
+            raise self.wrap_error(e=e)
+        self.log_upload_complete(file_path=path_str, destination=upload_path.as_posix())
 
     async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         path_str = str(path.resolve())
         upload_path = self.get_upload_path(file_data=file_data)
-        # Odd that fsspec doesn't run exists() as async even when client support async
-        logger.debug(f"writing local file {path_str} to {upload_path}")
-        with self.connection_config.get_client(protocol=self.upload_config.protocol) as client:
-            client.upload(lpath=path_str, rpath=upload_path.as_posix())
+        self.log_upload_start(file_path=path_str, destination=upload_path.as_posix())
+        try:
+            with self.connection_config.get_client(protocol=self.upload_config.protocol) as client:
+                client.upload(lpath=path_str, rpath=upload_path.as_posix())
+        except Exception as e:
+            self.log_error(
+                "File upload failed",
+                error=e,
+                context={"file_path": path_str, "destination": upload_path.as_posix()},
+            )
+            raise self.wrap_error(e=e)
+        self.log_upload_complete(file_path=path_str, destination=upload_path.as_posix())

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/processes/connectors/fsspec/s3.py

@@ -104,7 +104,13 @@ class S3ConnectionConfig(FsspecConnectionConfig):
                 return UserError(message)
             if http_code >= 500:
                 return ProviderError(message)
-        logger.error(f"unhandled exception from s3 ({type(e)}): {e}", exc_info=True)
+        logger.error(
+            "Unhandled exception from S3 (type: %s, endpoint: %s): %s",
+            type(e).__name__,
+            self.endpoint_url or "default",
+            e,
+            exc_info=True,
+        )
         return e
 
 
@@ -122,6 +128,10 @@ class S3Indexer(FsspecIndexer):
 
     def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
         path = file_info["Key"]
+
+        self.log_debug("Getting metadata for S3 object", context={"file_path": path})
+        self.log_file_operation("Getting metadata", file_path=path)
+
         date_created = None
         date_modified = None
         modified = file_info.get("LastModified")
@@ -147,9 +157,9 @@ class S3Indexer(FsspecIndexer):
             record_locator["metadata"] = metadata
         issue_characters = [char for char in CHARACTERS_TO_AVOID if char in path]
         if issue_characters:
-            logger.warning(
-                f"File path {path} contains characters "
-                f"that can cause issues with S3: {issue_characters}"
+            self.log_warning(
+                f"File path contains characters that can cause issues with S3: {issue_characters}",
+                context={"path": path, "problematic_characters": issue_characters},
             )
         return FileDataSourceMetadata(
             date_created=date_created,

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/processes/connectors/onedrive.py

@@ -115,23 +115,24 @@ class OnedriveConnectionConfig(ConnectionConfig):
             except ValueError as exc:
                 logger.error("Couldn't set up credentials.")
                 raise exc
-            
+
             if "error" in token:
                 error_codes = token.get("error_codes", [])
                 error_type = token.get("error", "")
                 error_description = token.get("error_description", "")
-                
+
                 # 7000215: Invalid client secret provided
                 # 7000218: Invalid client id provided
                 # 700016: Application not found in directory
                 # 90002: Tenant not found
                 auth_error_codes = [7000215, 7000218, 700016, 90002]
-                
-                if (any(code in error_codes for code in auth_error_codes) or 
-                    error_type in ["invalid_client", "unauthorized_client", "invalid_grant"]):
-                    raise UserAuthError(
-                        f"Authentication failed: {error_type}: {error_description}"
-                    )
+
+                if any(code in error_codes for code in auth_error_codes) or error_type in [
+                    "invalid_client",
+                    "unauthorized_client",
+                    "invalid_grant",
+                ]:
+                    raise UserAuthError(f"Authentication failed: {error_type}: {error_description}")
                 else:
                     raise SourceConnectionNetworkError(
                         f"Failed to fetch token: {error_type}: {error_description}"

{unstructured_ingest-1.0.56 → unstructured_ingest-1.1.2}/unstructured_ingest/processes/connectors/sharepoint.py

@@ -87,6 +87,7 @@ class SharepointIndexerConfig(OnedriveIndexerConfig):
     # TODO: We can probably make path non-optional on OnedriveIndexerConfig once tested
     path: str = Field(default="")
 
+
 @dataclass
 class SharepointIndexer(OnedriveIndexer):
     connection_config: SharepointConnectionConfig
@@ -114,14 +115,14 @@ class SharepointIndexer(OnedriveIndexer):
     def _is_root_path(self, path: str) -> bool:
         """Check if the path represents root access (empty string or legacy default)."""
         return not path or not path.strip() or path == LEGACY_DEFAULT_PATH
-    
+
     def _get_target_drive_item(self, site_drive_item: DriveItem, path: str) -> DriveItem:
         """Get the drive item to search in based on the path."""
         if self._is_root_path(path):
             return site_drive_item
         else:
             return site_drive_item.get_by_path(path).get().execute_query()
-    
+
     def _validate_folder_path(self, site_drive_item: DriveItem, path: str) -> None:
         """Validate that a specific folder path exists and is accessible."""
         from office365.runtime.client_request_exception import ClientRequestException

unstructured_ingest-1.1.2/unstructured_ingest/processes/utils/__init__.py

@@ -0,0 +1,8 @@
+from .logging.connector import ConnectorLoggingMixin, LoggingConfig
+from .logging.sanitizer import DataSanitizer
+
+__all__ = [
+    "ConnectorLoggingMixin",
+    "DataSanitizer",
+    "LoggingConfig",
+]