unstructured-ingest 0.0.1__py3-none-any.whl → 0.0.2.dev0__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.1"  # pragma: no cover
+__version__ = "0.0.2-dev0"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -7,6 +7,7 @@ from datetime import datetime
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
+from uuid import NAMESPACE_DNS, uuid5
 
 from unstructured.documents.elements import DataSourceMetadata
 
@@ -210,8 +211,11 @@ class FsspecIndexer(Indexer):
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
             rel_path = file.replace(self.index_config.path_without_protocol, "").lstrip("/")
+
+            additional_metadata = self.sterilize_info(path=file)
+            additional_metadata["original_file_path"] = file
             yield FileData(
-                identifier=file,
+                identifier=str(uuid5(NAMESPACE_DNS, file)),
                 connector_type=self.connector_type,
                 source_identifiers=SourceIdentifiers(
                     filename=Path(file).name,
@@ -219,7 +223,7 @@ class FsspecIndexer(Indexer):
                     fullpath=file,
                 ),
                 metadata=self.get_metadata(path=file),
-                additional_metadata=self.sterilize_info(path=file),
+                additional_metadata=additional_metadata,
             )
 
 
@@ -262,7 +266,8 @@ class FsspecDownloader(Downloader):
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         try:
-            self.fs.get(rpath=file_data.identifier, lpath=download_path.as_posix())
+            rpath = file_data.additional_metadata["original_file_path"]
+            self.fs.get(rpath=rpath, lpath=download_path.as_posix())
         except Exception as e:
             logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
             raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
@@ -272,7 +277,8 @@ class FsspecDownloader(Downloader):
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         try:
-            await self.fs.get(rpath=file_data.identifier, lpath=download_path.as_posix())
+            rpath = file_data.additional_metadata["original_file_path"]
+            await self.fs.get(rpath=rpath, lpath=download_path.as_posix())
         except Exception as e:
             logger.error(f"failed to download file {file_data.identifier}: {e}", exc_info=True)
             raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")

{unstructured_ingest-0.0.1.dist-info → unstructured_ingest-0.0.2.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.0.1
+Version: 0.0.2.dev0
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -21,9 +21,9 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
-Requires-Dist: pandas
 Requires-Dist: unstructured
 Requires-Dist: python-dateutil
+Requires-Dist: pandas
 Provides-Extra: airtable
 Requires-Dist: pyairtable ; extra == 'airtable'
 Provides-Extra: astra
@@ -39,8 +39,8 @@ Requires-Dist: boto3 ; extra == 'bedrock'
 Provides-Extra: biomed
 Requires-Dist: bs4 ; extra == 'biomed'
 Provides-Extra: box
-Requires-Dist: fsspec ; extra == 'box'
 Requires-Dist: boxfs ; extra == 'box'
+Requires-Dist: fsspec ; extra == 'box'
 Provides-Extra: chroma
 Requires-Dist: typer <=0.9.0 ; extra == 'chroma'
 Requires-Dist: chromadb ; extra == 'chroma'
@@ -63,30 +63,30 @@ Requires-Dist: unstructured[docx] ; extra == 'doc'
 Provides-Extra: docx
 Requires-Dist: unstructured[docx] ; extra == 'docx'
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs ; extra == 'dropbox'
 Requires-Dist: fsspec ; extra == 'dropbox'
+Requires-Dist: dropboxdrivefs ; extra == 'dropbox'
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async] ; extra == 'elasticsearch'
 Provides-Extra: embed-huggingface
-Requires-Dist: huggingface ; extra == 'embed-huggingface'
 Requires-Dist: langchain-community ; extra == 'embed-huggingface'
+Requires-Dist: huggingface ; extra == 'embed-huggingface'
 Requires-Dist: sentence-transformers ; extra == 'embed-huggingface'
 Provides-Extra: embed-octoai
 Requires-Dist: tiktoken ; extra == 'embed-octoai'
 Requires-Dist: openai ; extra == 'embed-octoai'
 Provides-Extra: embed-vertexai
-Requires-Dist: langchain-community ; extra == 'embed-vertexai'
 Requires-Dist: langchain ; extra == 'embed-vertexai'
 Requires-Dist: langchain-google-vertexai ; extra == 'embed-vertexai'
+Requires-Dist: langchain-community ; extra == 'embed-vertexai'
 Provides-Extra: embed-voyageai
-Requires-Dist: langchain-voyageai ; extra == 'embed-voyageai'
 Requires-Dist: langchain ; extra == 'embed-voyageai'
+Requires-Dist: langchain-voyageai ; extra == 'embed-voyageai'
 Provides-Extra: epub
 Requires-Dist: unstructured[epub] ; extra == 'epub'
 Provides-Extra: gcs
 Requires-Dist: bs4 ; extra == 'gcs'
-Requires-Dist: gcsfs ; extra == 'gcs'
 Requires-Dist: fsspec ; extra == 'gcs'
+Requires-Dist: gcsfs ; extra == 'gcs'
 Provides-Extra: github
 Requires-Dist: pygithub >1.58.0 ; extra == 'github'
 Provides-Extra: gitlab
@@ -109,18 +109,18 @@ Requires-Dist: pymongo ; extra == 'mongodb'
 Provides-Extra: msg
 Requires-Dist: unstructured[msg] ; extra == 'msg'
 Provides-Extra: notion
-Requires-Dist: htmlBuilder ; extra == 'notion'
 Requires-Dist: notion-client ; extra == 'notion'
+Requires-Dist: htmlBuilder ; extra == 'notion'
 Provides-Extra: odt
 Requires-Dist: unstructured[odt] ; extra == 'odt'
 Provides-Extra: onedrive
-Requires-Dist: bs4 ; extra == 'onedrive'
 Requires-Dist: Office365-REST-Python-Client ; extra == 'onedrive'
 Requires-Dist: msal ; extra == 'onedrive'
+Requires-Dist: bs4 ; extra == 'onedrive'
 Provides-Extra: openai
-Requires-Dist: tiktoken ; extra == 'openai'
 Requires-Dist: langchain-community ; extra == 'openai'
 Requires-Dist: openai ; extra == 'openai'
+Requires-Dist: tiktoken ; extra == 'openai'
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py ; extra == 'opensearch'
 Provides-Extra: org
@@ -147,13 +147,13 @@ Requires-Dist: unstructured[rst] ; extra == 'rst'
 Provides-Extra: rtf
 Requires-Dist: unstructured[rtf] ; extra == 'rtf'
 Provides-Extra: s3
-Requires-Dist: fsspec ; extra == 's3'
 Requires-Dist: s3fs ; extra == 's3'
+Requires-Dist: fsspec ; extra == 's3'
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce ; extra == 'salesforce'
 Provides-Extra: sftp
-Requires-Dist: paramiko ; extra == 'sftp'
 Requires-Dist: fsspec ; extra == 'sftp'
+Requires-Dist: paramiko ; extra == 'sftp'
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client ; extra == 'sharepoint'
 Requires-Dist: msal ; extra == 'sharepoint'

{unstructured_ingest-0.0.1.dist-info → unstructured_ingest-0.0.2.dev0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=SI019rW6paHw93e6fOWFzF9TruLom8o9HrgZsjGZvaE,42
+unstructured_ingest/__version__.py,sha256=neOBPct_gjgXqs6YN8HnzdaRiPiugEbJpwI6SDZ7qac,47
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/evaluate.py,sha256=R-mKLFXbVX1xQ1tjGsLHjdP-TbSSV-925IHzggW_bIg,9793
 unstructured_ingest/interfaces.py,sha256=uS8L5mS0mXD8I4XTfVlKZxAwqnpJ4yrRqn4vxWVRhQI,31107
@@ -346,13 +346,13 @@ unstructured_ingest/v2/processes/connectors/fsspec/__init__.py,sha256=TtdeImM7Yp
 unstructured_ingest/v2/processes/connectors/fsspec/azure.py,sha256=RN7zoifocIWVgoP9aMDMz4TP-Z9KhE-HbCCBq33fY90,4674
 unstructured_ingest/v2/processes/connectors/fsspec/box.py,sha256=UnD-F9g7yOOBStrAqeKq6GuQjEyHdwOA3jYLj8YZIRM,4088
 unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py,sha256=I6mPG9EIso9TcIczCw5Y14Yqd-EhTQ2CLw1MJx1V3dY,4420
-unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=gNgrRqKqk9YpBRGqGPvBUuEcBv1jN59fmBBj6NrB4sA,12394
+unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py,sha256=MgOUhDGtTAUuzmAsLvBwV_3ggyL5DDpMm-sb4KNck88,12689
 unstructured_ingest/v2/processes/connectors/fsspec/gcs.py,sha256=RYZq_8hKF7bRxuB5Gozv5AzB3_nTuuooE4UfRjXwEFU,4443
 unstructured_ingest/v2/processes/connectors/fsspec/s3.py,sha256=7lOm5hjb0LBkbe-OWXnV3wDC-3mM_GWwwmdKW0xzh8c,5333
 unstructured_ingest/v2/processes/connectors/fsspec/sftp.py,sha256=J7Ej-j7dtXAluHunwynUfHlNsYwymb-LsrGUFcljcsA,5700
 unstructured_ingest/v2/processes/connectors/fsspec/utils.py,sha256=jec_Qfe2hbfahBuY-u8FnvHuv933AI5HwPFjOL3kEEY,456
-unstructured_ingest-0.0.1.dist-info/METADATA,sha256=Qru27Cxrf0C-vFe7MqfaKOfavazrWYTTRif6loKf71o,21568
-unstructured_ingest-0.0.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-unstructured_ingest-0.0.1.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.0.1.dist-info/top_level.txt,sha256=QaTxTcjfM5Hr9sZJ6weOJvSe5ESQc0F8AWkhHInTCf8,20
-unstructured_ingest-0.0.1.dist-info/RECORD,,
+unstructured_ingest-0.0.2.dev0.dist-info/METADATA,sha256=dnWewzRLiYQlU6Fglws3oQnUkgzAVCnKM7BPMtls9YU,21573
+unstructured_ingest-0.0.2.dev0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+unstructured_ingest-0.0.2.dev0.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.0.2.dev0.dist-info/top_level.txt,sha256=QaTxTcjfM5Hr9sZJ6weOJvSe5ESQc0F8AWkhHInTCf8,20
+unstructured_ingest-0.0.2.dev0.dist-info/RECORD,,