unstructured-ingest 1.2.1__py3-none-any.whl → 1.2.3__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.1"  # pragma: no cover
+__version__ = "1.2.3"  # pragma: no cover

unstructured_ingest/interfaces/downloader.py

@@ -36,9 +36,11 @@ class Downloader(BaseProcess, BaseConnector, ABC):
     def get_download_path(self, file_data: FileData) -> Optional[Path]:
         if not file_data.source_identifiers:
             return None
+        
         rel_path = file_data.source_identifiers.relative_path
         if not rel_path:
             return None
+        
         rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
         return self.download_dir / Path(rel_path)
 

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -264,12 +264,31 @@ FsspecDownloaderConfigT = TypeVar("FsspecDownloaderConfigT", bound=FsspecDownloa
 
 @dataclass
 class FsspecDownloader(Downloader):
+    TEMP_DIR_PREFIX = "unstructured_"
+    
     protocol: str
     connection_config: FsspecConnectionConfigT
     connector_type: str = CONNECTOR_TYPE
     download_config: Optional[FsspecDownloaderConfigT] = field(
         default_factory=lambda: FsspecDownloaderConfig()
     )
+    
+    def get_download_path(self, file_data: FileData) -> Optional[Path]:
+        has_source_identifiers = file_data.source_identifiers is not None
+        has_filename = has_source_identifiers and file_data.source_identifiers.filename
+        
+        if not (has_source_identifiers and has_filename):
+            return None
+        
+        filename = file_data.source_identifiers.filename
+        
+        mkdir_concurrent_safe(self.download_dir)
+        
+        temp_dir = tempfile.mkdtemp(
+            prefix=self.TEMP_DIR_PREFIX, 
+            dir=self.download_dir
+        )
+        return Path(temp_dir) / filename
 
     def is_async(self) -> bool:
         with self.connection_config.get_client(protocol=self.protocol) as client:

unstructured_ingest/processes/connectors/fsspec/s3.py

@@ -1,4 +1,5 @@
 import contextlib
+import os
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from time import time
@@ -33,7 +34,7 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 
 CONNECTOR_TYPE = "s3"
 
-# https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines-avoid-characters
+# https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines-avoid-characters  # noqa
 CHARACTERS_TO_AVOID = ["\\", "{", "^", "}", "%", "`", "]", '"', ">", "[", "~", "<", "#", "|"]
 
 if TYPE_CHECKING:
@@ -56,6 +57,13 @@ class S3AccessConfig(FsspecAccessConfig):
     token: Optional[str] = Field(
         default=None, description="If not anonymous, use this security token, if specified."
     )
+    ambient_credentials: bool = Field(
+        default=False,
+        description="Explicitly allow using ambient AWS credentials from .aws folder, "
+        "environment variables, or IAM roles. Requires ALLOW_AMBIENT_CREDENTIALS_S3 environment "
+        "variable to also be set to 'true' (case insensitive) for security. When False (default), "
+        "only explicit credentials or anonymous access are allowed.",
+    )
 
 
 class S3ConnectionConfig(FsspecConnectionConfig):
@@ -72,14 +80,48 @@ class S3ConnectionConfig(FsspecConnectionConfig):
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
-        access_configs: dict[str, Any] = {"anon": self.anonymous}
+        access_config = self.access_config.get_secret_value()
+        has_explicit_credentials = bool(
+            access_config.key or access_config.secret or access_config.token
+        )
+
+        access_configs: dict[str, Any]
+
+        if has_explicit_credentials:
+            access_configs = {"anon": False}
+            # Avoid injecting None by filtering out k,v pairs where the value is None
+            access_configs.update(
+                {
+                    k: v
+                    for k, v in access_config.model_dump().items()
+                    if v is not None and k != "ambient_credentials"
+                }
+            )
+        elif access_config.ambient_credentials:
+            if os.getenv("ALLOW_AMBIENT_CREDENTIALS_S3", "").lower() == "true":
+                logger.info(
+                    "Using ambient AWS credentials (environment variables, .aws folder, IAM roles)"
+                )
+                access_configs = {"anon": False}
+                # Don't pass explicit credentials, let s3fs/boto3 auto-detect
+            else:
+                # Field allows but environment doesn't - raise error for security
+                raise UserAuthError(
+                    "Ambient credentials requested (ambient_credentials=True) but "
+                    "ALLOW_AMBIENT_CREDENTIALS_S3 environment variable is not set to 'true'. "
+                )
+        elif self.anonymous:
+            access_configs = {"anon": True}
+        else:
+            # User set anonymous=False but provided no credentials and no ambient permission
+            raise UserAuthError(
+                "No authentication method specified. anonymous=False but no explicit credentials "
+                "provided and ambient_credentials=False."
+            )
+
         if self.endpoint_url:
             access_configs["endpoint_url"] = self.endpoint_url
 
-        # Avoid injecting None by filtering out k,v pairs where the value is None
-        access_configs.update(
-            {k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v}
-        )
         return access_configs
 
     @requires_dependencies(["s3fs", "fsspec"], extras="s3")

{unstructured_ingest-1.2.1.dist-info → unstructured_ingest-1.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.2.1
+Version: 1.2.3
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.2.1.dist-info → unstructured_ingest-1.2.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=r2tk3QlR-3IlKjbYsMIts7ynZhCAyO4v-dzzULrlUCM,42
+unstructured_ingest/__version__.py,sha256=rNRejJU1rpN1Dw-S8aXOaPrt3Fh403mDtzJwDpLdB_Q,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -34,7 +34,7 @@ unstructured_ingest/embed/vertexai.py,sha256=DphvPhiYdXTMrQxJCd-64vMs4iVdLY_BphH
 unstructured_ingest/embed/voyageai.py,sha256=EOrYzaoXOZ6C4fNkMlCgb8KA8rdfgVXN3USMFpnn0Bs,4698
 unstructured_ingest/interfaces/__init__.py,sha256=QIkWqjsq9INTa89gPuXlMlQL4s3y5TqLmPkuVuTyXcs,795
 unstructured_ingest/interfaces/connector.py,sha256=wYWIEAL99KdQDDzzDYSf_yE8p1wjThSPMgEV5qyfiPc,1885
-unstructured_ingest/interfaces/downloader.py,sha256=xX0ZzsFRSzZb7SAeoeQph8sIbVq13DRw-3MYkdADrY0,2918
+unstructured_ingest/interfaces/downloader.py,sha256=7pJ4wpWrP645lgTx9dO0rni8chiCAjsnOFaXRtJe8IY,2936
 unstructured_ingest/interfaces/indexer.py,sha256=c2FwWJEQHfFD6vO-tGfYLpLiIs-TYViLAt8YmHfDbaM,824
 unstructured_ingest/interfaces/process.py,sha256=S3A_9gkwwGC-iQxvnpj3Er6IJAjAT5npzpSgxuFAzUM,449
 unstructured_ingest/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
@@ -109,9 +109,9 @@ unstructured_ingest/processes/connectors/fsspec/__init__.py,sha256=3HTdw4L4mdN4W
 unstructured_ingest/processes/connectors/fsspec/azure.py,sha256=31VNiG5YnXfhrFX7QJ2O1ubeWHxbe1sYVIztefbscAQ,7148
 unstructured_ingest/processes/connectors/fsspec/box.py,sha256=1gLS7xR2vbjgKBrQ4ZpI1fKTsJuIDfXuAzx_a4FzxG4,5873
 unstructured_ingest/processes/connectors/fsspec/dropbox.py,sha256=HwwKjQmjM7yFk9Esh_F20xDisRPXGUkFduzaasByRDE,8355
-unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=fA9jtXnr1P4wr8VBpZ1Lx9TsZzH-FDqHoBvPUH0DnWk,17827
+unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=yIvaII_uQ6ANibyj9aysM6c7fg5vUuL2eccLb51LhWk,18497
 unstructured_ingest/processes/connectors/fsspec/gcs.py,sha256=ouxISCKpZTAj3T6pWGYbASu93wytJjl5WSICvQcrgfE,7172
-unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=P5nd3hamhLFO3l5nV3lMuIxHtb_rZYFP4F6q_py3xpc,7492
+unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=UX1kZR_1RtMhhTxGfkXFRrNwFxDXgog8RG36qsqzZE0,9415
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
 unstructured_ingest/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
 unstructured_ingest/processes/connectors/ibm_watsonx/__init__.py,sha256=kf0UpgdAY2KK1R1FbAB6GEBBAIOeYQ8cZIr3bp660qM,374
@@ -235,8 +235,8 @@ unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3r
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
 unstructured_ingest/utils/tls.py,sha256=Ra8Mii1F4VqErRreg76PBI0eAqPBC009l0sSHa8FdnA,448
-unstructured_ingest-1.2.1.dist-info/METADATA,sha256=dABn7DHmV7FLVs7oG3G_ltcDE5OSERhhZdGfJAoDtN4,8826
-unstructured_ingest-1.2.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.2.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.2.1.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.2.1.dist-info/RECORD,,
+unstructured_ingest-1.2.3.dist-info/METADATA,sha256=pOJRa3rcHmixBKaIZGlem8vVYSpctM4jagYxyiprnn8,8826
+unstructured_ingest-1.2.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.2.3.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.2.3.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.2.3.dist-info/RECORD,,