unstructured-ingest 0.4.5__py3-none-any.whl → 0.4.6__py3-none-any.whl

test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -9,7 +9,8 @@ import pytest
 from databricks.sql import connect
 from databricks.sql.client import Connection as DeltaTableConnection
 from databricks.sql.client import Cursor as DeltaTableCursor
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel, Secret, SecretStr
+from pytest_mock import MockerFixture
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
 from test.integration.utils import requires_env
@@ -140,3 +141,30 @@ async def test_databricks_delta_tables_destination(
     uploader.precheck()
     uploader.run(path=staged_path, file_data=mock_file_data)
     validate_destination(expected_num_elements=expected_num_elements, table_name=destination_table)
+
+
+def test_get_credentials_provider_with_client_id_and_secret(mocker: MockerFixture):
+    access_config = DatabricksDeltaTablesAccessConfig(
+        client_id="test_client_id", client_secret="test_client_secret"
+    )
+    connection_config = DatabricksDeltaTablesConnectionConfig(
+        access_config=Secret(access_config),
+        server_hostname="test_server_hostname",
+        http_path="test_http_path",
+    )
+
+    credentials_provider = connection_config.get_credentials_provider()
+    assert credentials_provider is not False
+    assert type(credentials_provider).__name__ == "function"
+
+
+def test_get_credentials_provider_with_token(mocker: MockerFixture):
+    access_config = DatabricksDeltaTablesAccessConfig(token="test_token")
+    connection_config = DatabricksDeltaTablesConnectionConfig(
+        access_config=Secret(access_config),
+        server_hostname="test_server_hostname",
+        http_path="test_http_path",
+    )
+
+    credentials_provider = connection_config.get_credentials_provider()
+    assert credentials_provider is False

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.5"  # pragma: no cover
+__version__ = "0.4.6"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -42,7 +42,7 @@ if TYPE_CHECKING:
     from office365.onedrive.drives.drive import Drive
 
 CONNECTOR_TYPE = "onedrive"
-MAX_MB_SIZE = 512_000_000
+MAX_BYTES_SIZE = 512_000_000
 
 
 class OnedriveAccessConfig(AccessConfig):
@@ -251,7 +251,7 @@ class OnedriveDownloader(Downloader):
             download_path = self.get_download_path(file_data=file_data)
             download_path.parent.mkdir(parents=True, exist_ok=True)
             logger.info(f"downloading {file_data.source_identifiers.fullpath} to {download_path}")
-            if fsize > MAX_MB_SIZE:
+            if fsize > MAX_BYTES_SIZE:
                 logger.info(f"downloading file with size: {fsize} bytes in chunks")
                 with download_path.open(mode="wb") as f:
                     file.download_session(f, chunk_size=1024 * 1024 * 100).execute_query()
@@ -313,7 +313,7 @@ class OnedriveUploader(Uploader):
             try:
                 folder.get().execute_query()
             except ClientRequestException as e:
-                if e.message != "The resource could not be found.":
+                if not e.response.status_code == 404:
                     raise e
                 folder = root.create_folder(root_folder).execute_query()
                 logger.info(f"successfully created folder: {folder.name}")
@@ -321,7 +321,11 @@ class OnedriveUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
+    @requires_dependencies(["office365"], extras="onedrive")
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        from office365.onedrive.driveitems.conflict_behavior import ConflictBehavior
+        from office365.runtime.client_request_exception import ClientRequestException
+
         drive = self.connection_config.get_drive()
 
         # Use the remote_url from upload_config as the base destination folder
@@ -331,11 +335,11 @@ class OnedriveUploader(Uploader):
         if file_data.source_identifiers and file_data.source_identifiers.rel_path:
             # Combine the base destination folder with the file's relative path
             destination_path = Path(base_destination_folder) / Path(
-                file_data.source_identifiers.rel_path
+                f"{file_data.source_identifiers.rel_path}.json"
             )
         else:
             # If no relative path is provided, upload directly to the base destination folder
-            destination_path = Path(base_destination_folder) / path.name
+            destination_path = Path(base_destination_folder) / f"{path.name}.json"
 
         destination_folder = destination_path.parent
         file_name = destination_path.name
@@ -348,27 +352,19 @@ class OnedriveUploader(Uploader):
             # Attempt to get the folder
             folder = drive.root.get_by_path(destination_folder_str)
             folder.get().execute_query()
-        except Exception:
+        except ClientRequestException as e:
             # Folder doesn't exist, create it recursively
-            current_folder = drive.root
-            for part in destination_folder.parts:
-                # Use filter to find the folder by name
-                folders = (
-                    current_folder.children.filter(f"name eq '{part}' and folder ne null")
-                    .get()
-                    .execute_query()
-                )
-                if folders:
-                    current_folder = folders[0]
-                else:
-                    # Folder doesn't exist, create it
-                    current_folder = current_folder.create_folder(part).execute_query()
-            folder = current_folder
+            root = drive.root
+            root_folder = self.upload_config.root_folder
+            if not e.response.status_code == 404:
+                raise e
+            folder = root.create_folder(root_folder).execute_query()
+            logger.info(f"successfully created folder: {folder.name}")
 
         # Check the size of the file
         file_size = path.stat().st_size
 
-        if file_size < MAX_MB_SIZE:
+        if file_size < MAX_BYTES_SIZE:
             # Use simple upload for small files
             with path.open("rb") as local_file:
                 content = local_file.read()
@@ -388,19 +384,26 @@ class OnedriveUploader(Uploader):
                     ) from e
         else:
             # Use resumable upload for large files
-            destination_fullpath = f"{destination_folder_str}/{file_name}"
-            destination_drive_item = drive.root.item_with_path(destination_fullpath)
+            destination_drive_item = drive.root.get_by_path(destination_folder_str)
+
+            logger.info(
+                f"Uploading {path.parent / file_name} to {destination_folder_str} using resumable upload"  # noqa: E501
+            )
 
-            logger.info(f"Uploading {path} to {destination_fullpath} using resumable upload")
             try:
                 uploaded_file = destination_drive_item.resumable_upload(
                     source_path=str(path)
                 ).execute_query()
+                # Rename the uploaded file to the original source name with a .json extension
+                # Overwrite the file if it already exists
+                renamed_file = uploaded_file.move(
+                    name=file_name, conflict_behavior=ConflictBehavior.Replace
+                ).execute_query()
                 # Validate the upload
-                if not uploaded_file or uploaded_file.name != file_name:
+                if not renamed_file or renamed_file.name != file_name:
                     raise DestinationConnectionError(f"Upload failed for file '{file_name}'")
                 # Log details about the uploaded file
-                logger.info(f"Uploaded file {uploaded_file.name} with ID {uploaded_file.id}")
+                logger.info(f"Uploaded file {renamed_file.name} with ID {renamed_file.id}")
             except Exception as e:
                 logger.error(f"Failed to upload file '{file_name}' using resumable upload: {e}")
                 raise DestinationConnectionError(

unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -51,9 +51,10 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
 
         host = f"https://{self.server_hostname}"
         access_configs = self.access_config.get_secret_value()
-        if (client_id := access_configs.client_id) and (
-            client_secret := access_configs.client_secret
-        ):
+        client_id = access_configs.client_id
+        client_secret = access_configs.client_secret
+
+        def _get_credentials_provider():
             return oauth_service_principal(
                 Config(
                     host=host,
@@ -61,6 +62,10 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
                     client_secret=client_secret,
                 )
             )
+
+        if client_id and client_secret:
+            return _get_credentials_provider
+
         return False
 
     def model_post_init(self, __context: Any) -> None:

{unstructured_ingest-0.4.5.dist-info → unstructured_ingest-0.4.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.4.5
+Version: 0.4.6
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,19 +23,19 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: pydantic>=2.7
-Requires-Dist: pandas
-Requires-Dist: tqdm
+Requires-Dist: dataclasses-json
 Requires-Dist: python-dateutil
 Requires-Dist: click
-Requires-Dist: dataclasses-json
+Requires-Dist: tqdm
+Requires-Dist: pandas
 Requires-Dist: opentelemetry-sdk
 Provides-Extra: airtable
 Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: bedrock
@@ -45,8 +45,8 @@ Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -63,8 +63,8 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: databricks-volumes
 Requires-Dist: databricks-sdk; extra == "databricks-volumes"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: doc
@@ -72,8 +72,8 @@ Requires-Dist: unstructured[docx]; extra == "doc"
 Provides-Extra: docx
 Requires-Dist: unstructured[docx]; extra == "docx"
 Provides-Extra: dropbox
-Requires-Dist: fsspec; extra == "dropbox"
 Requires-Dist: dropboxdrivefs; extra == "dropbox"
+Requires-Dist: fsspec; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
@@ -93,8 +93,8 @@ Provides-Extra: epub
 Requires-Dist: unstructured[epub]; extra == "epub"
 Provides-Extra: gcs
 Requires-Dist: bs4; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
@@ -103,8 +103,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -122,20 +122,20 @@ Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: msg
 Requires-Dist: unstructured[msg]; extra == "msg"
 Provides-Extra: neo4j
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: odt
 Requires-Dist: unstructured[odt]; extra == "odt"
 Provides-Extra: onedrive
 Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: openai
 Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
@@ -144,8 +144,8 @@ Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: org
 Requires-Dist: unstructured[org]; extra == "org"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pdf
 Requires-Dist: unstructured[pdf]; extra == "pdf"
 Provides-Extra: pinecone
@@ -174,11 +174,11 @@ Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: slack
@@ -195,9 +195,9 @@ Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: vectara
+Requires-Dist: requests; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
-Requires-Dist: requests; extra == "vectara"
 Provides-Extra: weaviate
 Requires-Dist: weaviate-client; extra == "weaviate"
 Provides-Extra: wikipedia

{unstructured_ingest-0.4.5.dist-info → unstructured_ingest-0.4.6.dist-info}/RECORD

@@ -34,7 +34,7 @@ test/integration/connectors/elasticsearch/conftest.py,sha256=-i4_7MkIxSQENz7nuD2
 test/integration/connectors/elasticsearch/test_elasticsearch.py,sha256=TsSEPsyaTUoEvFBadinrdM0b5C4FoUtEwCv24OUbpO8,12072
 test/integration/connectors/elasticsearch/test_opensearch.py,sha256=7b7z0GqoBsBqA3IK35N6axmwEMjzJ1l3Fg2WT2c7uqs,11450
 test/integration/connectors/sql/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-test/integration/connectors/sql/test_databricks_delta_tables.py,sha256=aC8B7peVYR8L2QaR18arqR6ffA197IsVkM2quCOVNSo,5046
+test/integration/connectors/sql/test_databricks_delta_tables.py,sha256=qHRHrvh5cCdtTFSLM1bDwRnNgtBQetbdnyRnU9qvK0Y,6144
 test/integration/connectors/sql/test_postgres.py,sha256=bGDyzLRpgrXO7nl0U8nF2zSNr6ykUG-w8T4daIqUCG4,6970
 test/integration/connectors/sql/test_singlestore.py,sha256=XeU2s4Kt_3tGyaDYYKTgYjdOyb8j2dnz4TgSMwFUjWs,6153
 test/integration/connectors/sql/test_snowflake.py,sha256=LEwsRDoC6-rRiwYsqeo5B9Eo6RYygLLGAUsrtrgI9pM,7494
@@ -102,7 +102,7 @@ test/unit/v2/partitioners/test_partitioner.py,sha256=iIYg7IpftV3LusoO4H8tr1IHY1U
 test/unit/v2/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 test/unit/v2/utils/data_generator.py,sha256=UoYVNjG4S4wlaA9gceQ82HIpF9_6I1UTHD1_GrQBHp0,973
 unstructured_ingest/__init__.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
-unstructured_ingest/__version__.py,sha256=LZI8wLYHcTzImgX-mBT2GEDfuLfBbZawJ40Z_jZShYc,42
+unstructured_ingest/__version__.py,sha256=0ZfnDBlBmcWgua3sGv2Fwo28JBX-eiHGLg4rl98g_F0,42
 unstructured_ingest/error.py,sha256=qDncnJgbf5ils956RcO2CGlAKYDT5OaEM9Clv1JVTNc,1448
 unstructured_ingest/interfaces.py,sha256=OYVUP0bzBJpT-Lz92BDyz_hLBvyfxkuSwWHhUdnUayA,31493
 unstructured_ingest/logger.py,sha256=S5nSqGcABoQyeicgRnBQFjDScCaTvFVivOCvbo-laL0,4479
@@ -433,7 +433,7 @@ unstructured_ingest/v2/processes/connectors/local.py,sha256=ZvWTj6ZYkwnvQMNFsZWo
 unstructured_ingest/v2/processes/connectors/milvus.py,sha256=wmcu9NVy3gYlQGT25inN5w_QrhFoL8-hRq0pJFSNw8g,8866
 unstructured_ingest/v2/processes/connectors/mongodb.py,sha256=cL0QUQZF_s2brh3nNNeAywXVpaIiND4b5JTAFlYjLjw,14273
 unstructured_ingest/v2/processes/connectors/neo4j.py,sha256=HU1IwchTM7Q1kkeIFVe-Lg6gInMItBpgkDkVwuTvkGY,14259
-unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=9UK5nILtrAXSwpp_aeANgqvQf_UbH3J3czN7y-DL9d0,17386
+unstructured_ingest/v2/processes/connectors/onedrive.py,sha256=b616B_-9MfU6gxvpw7IBUa2szNFURA_VP8q5j2FXxnA,17632
 unstructured_ingest/v2/processes/connectors/outlook.py,sha256=KgNGM8hImRhy6_SpswRP2VwRD4VOrqqJoySgxf2oduI,9290
 unstructured_ingest/v2/processes/connectors/pinecone.py,sha256=bQDCch7OGiQgpWO3n3ncLuQ4XCWqDc7ZWEB-Qrqkss8,10730
 unstructured_ingest/v2/processes/connectors/redisdb.py,sha256=p0AY4ukBNpwAemV4bWzpScvVbLTVlI3DzsCNUKiBI5M,6757
@@ -549,7 +549,7 @@ unstructured_ingest/v2/processes/connectors/qdrant/local.py,sha256=cGEyv3Oy6y4BQ
 unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py,sha256=BHI7HYSdbS05j2vrjyDvLzVG1WfsM8osKeq-lttlybQ,5437
 unstructured_ingest/v2/processes/connectors/qdrant/server.py,sha256=odvCZWZp8DmRxLXMR7tHhW-c7UQbix1_zpFdfXfCvKI,1613
 unstructured_ingest/v2/processes/connectors/sql/__init__.py,sha256=NSEZwJDHh_9kFc31LnG14iRtYF3meK2UfUlQfYnwYEQ,2059
-unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=SRNplobVKd8fSeauYbLzBNlMb3HRHhinFS281B8aYtY,8854
+unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py,sha256=xbZ90rmehiCnBoqFXMz-3ZMXeYb0PzWB6iobCNSHTmQ,8955
 unstructured_ingest/v2/processes/connectors/sql/postgres.py,sha256=BATfX1PQGT2kl8jAbdNKXTojYKJxh3pJV9-h3OBnHGo,5124
 unstructured_ingest/v2/processes/connectors/sql/singlestore.py,sha256=OPBDQ2c_5KjWHEFfqXxf3pQ2tWC-N4MtslMulMgP1Wc,5503
 unstructured_ingest/v2/processes/connectors/sql/snowflake.py,sha256=QE-WBqrPVjCgcxR5EdVD9iTHBjgDSSSQgWYvq5N61qU,7746
@@ -561,9 +561,9 @@ unstructured_ingest/v2/processes/connectors/weaviate/cloud.py,sha256=bXtfEYLquR-
 unstructured_ingest/v2/processes/connectors/weaviate/embedded.py,sha256=S8Zg8StuZT-k7tCg1D5YShO1-vJYYk9-M1bE1fIqx64,3014
 unstructured_ingest/v2/processes/connectors/weaviate/local.py,sha256=LuTBKPseVewsz8VqxRPRLfGEm3BeI9nBZxpy7ZU5tOA,2201
 unstructured_ingest/v2/processes/connectors/weaviate/weaviate.py,sha256=yJza_jBSEFnzZRq5L6vJ0Mm3uS1uxkOiKIimPpUyQds,12418
-unstructured_ingest-0.4.5.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
-unstructured_ingest-0.4.5.dist-info/METADATA,sha256=cetNdLOmsQvHFt7j2m2utZKATnaMx9BIAV2i386aoTc,8051
-unstructured_ingest-0.4.5.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-unstructured_ingest-0.4.5.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
-unstructured_ingest-0.4.5.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
-unstructured_ingest-0.4.5.dist-info/RECORD,,
+unstructured_ingest-0.4.6.dist-info/LICENSE.md,sha256=SxkKP_62uIAKb9mb1eH7FH4Kn2aYT09fgjKpJt5PyTk,11360
+unstructured_ingest-0.4.6.dist-info/METADATA,sha256=-Z6UDd_I1lUsEbYTmeBlNb4D4-e3y67LM4n75igK1tY,8051
+unstructured_ingest-0.4.6.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+unstructured_ingest-0.4.6.dist-info/entry_points.txt,sha256=gUAAFnjFPnBgThJSEbw0N5ZjxtaKlT1s9e05_arQrNw,70
+unstructured_ingest-0.4.6.dist-info/top_level.txt,sha256=DMuDMHZRMdeay8v8Kdi855muIv92F0OkutvBCaBEW6M,25
+unstructured_ingest-0.4.6.dist-info/RECORD,,