unstructured-ingest 1.2.0__py3-none-any.whl → 1.2.2__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.0"  # pragma: no cover
+__version__ = "1.2.2"  # pragma: no cover

unstructured_ingest/embed/interfaces.py

@@ -67,14 +67,14 @@ class BaseEmbeddingEncoder(BaseEncoder, ABC):
         elements = elements.copy()
         elements_with_text = [e for e in elements if e.get("text")]
         texts = [e["text"] for e in elements_with_text]
-        embeddings = []
+        all_embeddings = []
         try:
             for batch in batch_generator(texts, batch_size=self.config.batch_size or len(texts)):
-                embeddings = self.embed_batch(client=client, batch=batch)
-                embeddings.extend(embeddings)
+                embeddings_batch = self.embed_batch(client=client, batch=batch)
+                all_embeddings.extend(embeddings_batch)
         except Exception as e:
             raise self.wrap_error(e=e)
-        for element, embedding in zip(elements_with_text, embeddings):
+        for element, embedding in zip(elements_with_text, all_embeddings, strict=True):
             element[EMBEDDINGS_KEY] = embedding
         return elements
 
@@ -123,14 +123,14 @@ class AsyncBaseEmbeddingEncoder(BaseEncoder, ABC):
         elements = elements.copy()
         elements_with_text = [e for e in elements if e.get("text")]
         texts = [e["text"] for e in elements_with_text]
-        embeddings = []
+        all_embeddings = []
         try:
             for batch in batch_generator(texts, batch_size=self.config.batch_size or len(texts)):
-                embeddings = await self.embed_batch(client=client, batch=batch)
-                embeddings.extend(embeddings)
+                embeddings_batch = await self.embed_batch(client=client, batch=batch)
+                all_embeddings.extend(embeddings_batch)
         except Exception as e:
             raise self.wrap_error(e=e)
-        for element, embedding in zip(elements_with_text, embeddings):
+        for element, embedding in zip(elements_with_text, all_embeddings, strict=True):
             element[EMBEDDINGS_KEY] = embedding
         return elements
 

unstructured_ingest/interfaces/downloader.py

@@ -36,9 +36,11 @@ class Downloader(BaseProcess, BaseConnector, ABC):
     def get_download_path(self, file_data: FileData) -> Optional[Path]:
         if not file_data.source_identifiers:
             return None
+        
         rel_path = file_data.source_identifiers.relative_path
         if not rel_path:
             return None
+        
         rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
         return self.download_dir / Path(rel_path)
 

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -264,12 +264,31 @@ FsspecDownloaderConfigT = TypeVar("FsspecDownloaderConfigT", bound=FsspecDownloa
 
 @dataclass
 class FsspecDownloader(Downloader):
+    TEMP_DIR_PREFIX = "unstructured_"
+    
     protocol: str
     connection_config: FsspecConnectionConfigT
     connector_type: str = CONNECTOR_TYPE
     download_config: Optional[FsspecDownloaderConfigT] = field(
         default_factory=lambda: FsspecDownloaderConfig()
     )
+    
+    def get_download_path(self, file_data: FileData) -> Optional[Path]:
+        has_source_identifiers = file_data.source_identifiers is not None
+        has_filename = has_source_identifiers and file_data.source_identifiers.filename
+        
+        if not (has_source_identifiers and has_filename):
+            return None
+        
+        filename = file_data.source_identifiers.filename
+        
+        mkdir_concurrent_safe(self.download_dir)
+        
+        temp_dir = tempfile.mkdtemp(
+            prefix=self.TEMP_DIR_PREFIX, 
+            dir=self.download_dir
+        )
+        return Path(temp_dir) / filename
 
     def is_async(self) -> bool:
         with self.connection_config.get_client(protocol=self.protocol) as client:

{unstructured_ingest-1.2.0.dist-info → unstructured_ingest-1.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.2.0
+Version: 1.2.2
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.2.0.dist-info → unstructured_ingest-1.2.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=hYDudzvEd17dAmuuGVXnxBsg1JfgQBVtuuoFz06SeYo,42
+unstructured_ingest/__version__.py,sha256=HPy7TMxiKrkQS-Rrw57HuZN3ZHBCTvYH8fjgFH1cXxs,41
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/errors_v2.py,sha256=9RuRCi7lbDxCguDz07y5RiHoQiFIOWwOD7xqzJ2B3Yw,436
 unstructured_ingest/logger.py,sha256=7e_7UeK6hVOd5BQ6i9NzRUAPCS_DF839Y8TjUDywraY,1428
@@ -25,7 +25,7 @@ unstructured_ingest/embed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 unstructured_ingest/embed/azure_openai.py,sha256=Q_buBkAcx9FBuTsAqKbRU8vd9vDh8JoDOEth4fFxHbg,2160
 unstructured_ingest/embed/bedrock.py,sha256=dzfCsatB0i8hUp1YnXmoImoxgvUdZ4srKI6eSvn-lYM,9132
 unstructured_ingest/embed/huggingface.py,sha256=6Gx9L3xa3cv9fX4AMuLsePJQF4T_jwkKjovfqF5X1NM,2435
-unstructured_ingest/embed/interfaces.py,sha256=Y3PLhgWnMDmtpugE37hlAiBIbC8izrFFXXkrPVby-HY,5137
+unstructured_ingest/embed/interfaces.py,sha256=VCrCSJiEfIxKB4NL4AHgKb-0vB_SEekb47zMUW6gWf0,5211
 unstructured_ingest/embed/mixedbreadai.py,sha256=uKTqzoi4M_WeYZu-qc_TSxwJONOESzxVbBLUbD1Wbns,3922
 unstructured_ingest/embed/octoai.py,sha256=yZuD7R4mEKS4Jjyae_IrNWogMPOFFS8gW5oUllj3ROU,4540
 unstructured_ingest/embed/openai.py,sha256=09I5BIrb-iGsv92LOV46-F7oZ7j1JnJIOQFARNKVq3k,5029
@@ -34,7 +34,7 @@ unstructured_ingest/embed/vertexai.py,sha256=DphvPhiYdXTMrQxJCd-64vMs4iVdLY_BphH
 unstructured_ingest/embed/voyageai.py,sha256=EOrYzaoXOZ6C4fNkMlCgb8KA8rdfgVXN3USMFpnn0Bs,4698
 unstructured_ingest/interfaces/__init__.py,sha256=QIkWqjsq9INTa89gPuXlMlQL4s3y5TqLmPkuVuTyXcs,795
 unstructured_ingest/interfaces/connector.py,sha256=wYWIEAL99KdQDDzzDYSf_yE8p1wjThSPMgEV5qyfiPc,1885
-unstructured_ingest/interfaces/downloader.py,sha256=xX0ZzsFRSzZb7SAeoeQph8sIbVq13DRw-3MYkdADrY0,2918
+unstructured_ingest/interfaces/downloader.py,sha256=7pJ4wpWrP645lgTx9dO0rni8chiCAjsnOFaXRtJe8IY,2936
 unstructured_ingest/interfaces/indexer.py,sha256=c2FwWJEQHfFD6vO-tGfYLpLiIs-TYViLAt8YmHfDbaM,824
 unstructured_ingest/interfaces/process.py,sha256=S3A_9gkwwGC-iQxvnpj3Er6IJAjAT5npzpSgxuFAzUM,449
 unstructured_ingest/interfaces/processor.py,sha256=VX7JqXlbG1plxMK8THWhWINPbTICaaUEk4XUXhnOixY,3303
@@ -109,7 +109,7 @@ unstructured_ingest/processes/connectors/fsspec/__init__.py,sha256=3HTdw4L4mdN4W
 unstructured_ingest/processes/connectors/fsspec/azure.py,sha256=31VNiG5YnXfhrFX7QJ2O1ubeWHxbe1sYVIztefbscAQ,7148
 unstructured_ingest/processes/connectors/fsspec/box.py,sha256=1gLS7xR2vbjgKBrQ4ZpI1fKTsJuIDfXuAzx_a4FzxG4,5873
 unstructured_ingest/processes/connectors/fsspec/dropbox.py,sha256=HwwKjQmjM7yFk9Esh_F20xDisRPXGUkFduzaasByRDE,8355
-unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=fA9jtXnr1P4wr8VBpZ1Lx9TsZzH-FDqHoBvPUH0DnWk,17827
+unstructured_ingest/processes/connectors/fsspec/fsspec.py,sha256=yIvaII_uQ6ANibyj9aysM6c7fg5vUuL2eccLb51LhWk,18497
 unstructured_ingest/processes/connectors/fsspec/gcs.py,sha256=ouxISCKpZTAj3T6pWGYbASu93wytJjl5WSICvQcrgfE,7172
 unstructured_ingest/processes/connectors/fsspec/s3.py,sha256=P5nd3hamhLFO3l5nV3lMuIxHtb_rZYFP4F6q_py3xpc,7492
 unstructured_ingest/processes/connectors/fsspec/sftp.py,sha256=pR_a2SgLjt8ffNkariHrPB1E0HVSTj5h3pt7KxTU3TI,6371
@@ -235,8 +235,8 @@ unstructured_ingest/utils/pydantic_models.py,sha256=BT_j15e4rX40wQbt8LUXbqfPhA3r
 unstructured_ingest/utils/string_and_date_utils.py,sha256=oXOI6rxXq-8ncbk7EoJK0WCcTXWj75EzKl8pfQMID3U,2522
 unstructured_ingest/utils/table.py,sha256=WZechczgVFvlodUWFcsnCGvBNh1xRm6hr0VbJTPxKAc,3669
 unstructured_ingest/utils/tls.py,sha256=Ra8Mii1F4VqErRreg76PBI0eAqPBC009l0sSHa8FdnA,448
-unstructured_ingest-1.2.0.dist-info/METADATA,sha256=3D67Gk9trwGIVvMh0oSUDx_aJDsYCHD0qwL1VB9ZoYw,8826
-unstructured_ingest-1.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-unstructured_ingest-1.2.0.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-1.2.0.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-1.2.0.dist-info/RECORD,,
+unstructured_ingest-1.2.2.dist-info/METADATA,sha256=kLg62BHEhhU0BK_73Qc0XqsKtrf5XN3pzD40eGXW3xM,8826
+unstructured_ingest-1.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+unstructured_ingest-1.2.2.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-1.2.2.dist-info/licenses/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-1.2.2.dist-info/RECORD,,