unstructured-ingest 0.4.5__tar.gz → 0.4.7__tar.gz

unstructured_ingest-0.4.7/MANIFEST.in

@@ -0,0 +1,2 @@
+# Connector specific assets
+include unstructured_ingest/v2/processes/connectors/assets/*

{unstructured_ingest-0.4.5/unstructured_ingest.egg-info → unstructured_ingest-0.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.4.5
+Version: 0.4.7
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: click
+Requires-Dist: tqdm
+Requires-Dist: pandas
 Requires-Dist: pydantic>=2.7
 Requires-Dist: dataclasses_json
 Requires-Dist: python-dateutil
-Requires-Dist: click
-Requires-Dist: pandas
-Requires-Dist: tqdm
-Requires-Dist: opentelemetry-sdk
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -74,15 +74,15 @@ Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
@@ -98,19 +98,19 @@ Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: fsspec; extra == "gcs"
-Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
+Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,23 +124,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: networkx; extra == "neo4j"
-Requires-Dist: neo4j; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j; extra == "neo4j"
+Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -152,21 +152,21 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: fsspec; extra == "s3"
 Requires-Dist: s3fs; extra == "s3"
+Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,18 +178,18 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
+Requires-Dist: httpx; extra == "vectara"
 Provides-Extra: vastdb
+Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"
-Requires-Dist: vastdb; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/setup.py

@@ -204,4 +204,5 @@ setup(
     extras_require=extras_require,
     package_dir={"unstructured_ingest": "unstructured_ingest"},
     package_data={"unstructured_ingest": ["py.typed"]},
+    include_package_data=True,
 )

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -9,7 +9,8 @@ import pytest
 from databricks.sql import connect
 from databricks.sql.client import Connection as DeltaTableConnection
 from databricks.sql.client import Cursor as DeltaTableCursor
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel, Secret, SecretStr
+from pytest_mock import MockerFixture
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
 from test.integration.utils import requires_env
@@ -140,3 +141,30 @@ async def test_databricks_delta_tables_destination(
     uploader.precheck()
     uploader.run(path=staged_path, file_data=mock_file_data)
     validate_destination(expected_num_elements=expected_num_elements, table_name=destination_table)
+
+
+def test_get_credentials_provider_with_client_id_and_secret(mocker: MockerFixture):
+    access_config = DatabricksDeltaTablesAccessConfig(
+        client_id="test_client_id", client_secret="test_client_secret"
+    )
+    connection_config = DatabricksDeltaTablesConnectionConfig(
+        access_config=Secret(access_config),
+        server_hostname="test_server_hostname",
+        http_path="test_http_path",
+    )
+
+    credentials_provider = connection_config.get_credentials_provider()
+    assert credentials_provider is not False
+    assert type(credentials_provider).__name__ == "function"
+
+
+def test_get_credentials_provider_with_token(mocker: MockerFixture):
+    access_config = DatabricksDeltaTablesAccessConfig(token="test_token")
+    connection_config = DatabricksDeltaTablesConnectionConfig(
+        access_config=Secret(access_config),
+        server_hostname="test_server_hostname",
+        http_path="test_http_path",
+    )
+
+    credentials_provider = connection_config.get_credentials_provider()
+    assert credentials_provider is False

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/test/integration/connectors/test_onedrive.py

@@ -18,9 +18,6 @@ from unstructured_ingest.v2.processes.connectors.onedrive import (
 
 
 @pytest.fixture
-@pytest.mark.xfail(
-    reason="Issues with test setup on the provider side."
-)  # TODO: remove line when issues are addressed
 def onedrive_test_folder() -> str:
     """
     Pytest fixture that creates a test folder in OneDrive and deletes it after test run.
@@ -67,9 +64,6 @@ def get_connection_config():
 
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
-@pytest.mark.xfail(
-    reason="Issues with test setup on the provider side."
-)  # TODO: remove line when issues are addressed
 def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
     """
     Integration test for the OneDrive destination connector.
@@ -107,10 +101,14 @@ def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
     client = connection_config.get_client()
     drive = client.users[user_pname].drive
 
+    # Workaround: File should not have .json in the metadata.filename it comes from embedder
     uploaded_file = (
-        drive.root.get_by_path(destination_fullpath).select(["id", "name"]).get().execute_query()
+        drive.root.get_by_path(f"{destination_fullpath}.json")
+        .select(["id", "name"])
+        .get()
+        .execute_query()
     )
 
     # Check if the file exists
     assert uploaded_file is not None
-    assert uploaded_file.name == upload_file.name
+    assert uploaded_file.name == f"{upload_file.name}.json"

unstructured_ingest-0.4.7/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.4.7"  # pragma: no cover

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/unstructured_ingest/embed/azure_openai.py

@@ -3,11 +3,15 @@ from typing import TYPE_CHECKING
 
 from pydantic import Field
 
-from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+from unstructured_ingest.embed.openai import (
+    AsyncOpenAIEmbeddingEncoder,
+    OpenAIEmbeddingConfig,
+    OpenAIEmbeddingEncoder,
+)
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
-    from openai import AzureOpenAI
+    from openai import AsyncAzureOpenAI, AzureOpenAI
 
 
 class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
@@ -25,7 +29,22 @@ class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
             azure_endpoint=self.azure_endpoint,
         )
 
+    @requires_dependencies(["openai"], extras="openai")
+    def get_async_client(self) -> "AsyncAzureOpenAI":
+        from openai import AsyncAzureOpenAI
+
+        return AsyncAzureOpenAI(
+            api_key=self.api_key.get_secret_value(),
+            api_version=self.api_version,
+            azure_endpoint=self.azure_endpoint,
+        )
+
 
 @dataclass
 class AzureOpenAIEmbeddingEncoder(OpenAIEmbeddingEncoder):
     config: AzureOpenAIEmbeddingConfig
+
+
+@dataclass
+class AsyncAzureOpenAIEmbeddingEncoder(AsyncOpenAIEmbeddingEncoder):
+    config: AzureOpenAIEmbeddingConfig

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/unstructured_ingest/v2/pipeline/pipeline.py

@@ -329,9 +329,9 @@ class Pipeline:
         source_entry = {
             k: v
             for k, v in source_registry.items()
-            if isinstance(indexer_config, v.indexer_config)
-            and isinstance(downloader_config, v.downloader_config)
-            and isinstance(source_connection_config, v.connection_config)
+            if type(indexer_config) is v.indexer_config
+            and type(downloader_config) is v.downloader_config
+            and type(source_connection_config) is v.connection_config
         }
         if len(source_entry) > 1:
             raise ValueError(

unstructured_ingest-0.4.7/unstructured_ingest/v2/processes/connectors/assets/weaviate_collection_config.json

@@ -0,0 +1,23 @@
+{
+    "properties": [
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "record_id",
+            "tokenization": "word"
+        },
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "text",
+            "tokenization": "word"
+        }
+    ],
+    "vectorizer": "none"
+}

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -42,7 +42,7 @@ if TYPE_CHECKING:
     from office365.onedrive.drives.drive import Drive
 
 CONNECTOR_TYPE = "onedrive"
-MAX_MB_SIZE = 512_000_000
+MAX_BYTES_SIZE = 512_000_000
 
 
 class OnedriveAccessConfig(AccessConfig):
@@ -251,7 +251,7 @@ class OnedriveDownloader(Downloader):
             download_path = self.get_download_path(file_data=file_data)
             download_path.parent.mkdir(parents=True, exist_ok=True)
             logger.info(f"downloading {file_data.source_identifiers.fullpath} to {download_path}")
-            if fsize > MAX_MB_SIZE:
+            if fsize > MAX_BYTES_SIZE:
                 logger.info(f"downloading file with size: {fsize} bytes in chunks")
                 with download_path.open(mode="wb") as f:
                     file.download_session(f, chunk_size=1024 * 1024 * 100).execute_query()
@@ -313,7 +313,7 @@ class OnedriveUploader(Uploader):
             try:
                 folder.get().execute_query()
             except ClientRequestException as e:
-                if e.message != "The resource could not be found.":
+                if not e.response.status_code == 404:
                     raise e
                 folder = root.create_folder(root_folder).execute_query()
                 logger.info(f"successfully created folder: {folder.name}")
@@ -321,7 +321,11 @@ class OnedriveUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
+    @requires_dependencies(["office365"], extras="onedrive")
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        from office365.onedrive.driveitems.conflict_behavior import ConflictBehavior
+        from office365.runtime.client_request_exception import ClientRequestException
+
         drive = self.connection_config.get_drive()
 
         # Use the remote_url from upload_config as the base destination folder
@@ -331,11 +335,11 @@ class OnedriveUploader(Uploader):
         if file_data.source_identifiers and file_data.source_identifiers.rel_path:
             # Combine the base destination folder with the file's relative path
             destination_path = Path(base_destination_folder) / Path(
-                file_data.source_identifiers.rel_path
+                f"{file_data.source_identifiers.rel_path}.json"
             )
         else:
             # If no relative path is provided, upload directly to the base destination folder
-            destination_path = Path(base_destination_folder) / path.name
+            destination_path = Path(base_destination_folder) / f"{path.name}.json"
 
         destination_folder = destination_path.parent
         file_name = destination_path.name
@@ -348,27 +352,19 @@ class OnedriveUploader(Uploader):
             # Attempt to get the folder
             folder = drive.root.get_by_path(destination_folder_str)
             folder.get().execute_query()
-        except Exception:
+        except ClientRequestException as e:
             # Folder doesn't exist, create it recursively
-            current_folder = drive.root
-            for part in destination_folder.parts:
-                # Use filter to find the folder by name
-                folders = (
-                    current_folder.children.filter(f"name eq '{part}' and folder ne null")
-                    .get()
-                    .execute_query()
-                )
-                if folders:
-                    current_folder = folders[0]
-                else:
-                    # Folder doesn't exist, create it
-                    current_folder = current_folder.create_folder(part).execute_query()
-            folder = current_folder
+            root = drive.root
+            root_folder = self.upload_config.root_folder
+            if not e.response.status_code == 404:
+                raise e
+            folder = root.create_folder(root_folder).execute_query()
+            logger.info(f"successfully created folder: {folder.name}")
 
         # Check the size of the file
         file_size = path.stat().st_size
 
-        if file_size < MAX_MB_SIZE:
+        if file_size < MAX_BYTES_SIZE:
             # Use simple upload for small files
             with path.open("rb") as local_file:
                 content = local_file.read()
@@ -388,19 +384,26 @@ class OnedriveUploader(Uploader):
                     ) from e
         else:
             # Use resumable upload for large files
-            destination_fullpath = f"{destination_folder_str}/{file_name}"
-            destination_drive_item = drive.root.item_with_path(destination_fullpath)
+            destination_drive_item = drive.root.get_by_path(destination_folder_str)
+
+            logger.info(
+                f"Uploading {path.parent / file_name} to {destination_folder_str} using resumable upload"  # noqa: E501
+            )
 
-            logger.info(f"Uploading {path} to {destination_fullpath} using resumable upload")
             try:
                 uploaded_file = destination_drive_item.resumable_upload(
                     source_path=str(path)
                 ).execute_query()
+                # Rename the uploaded file to the original source name with a .json extension
+                # Overwrite the file if it already exists
+                renamed_file = uploaded_file.move(
+                    name=file_name, conflict_behavior=ConflictBehavior.Replace
+                ).execute_query()
                 # Validate the upload
-                if not uploaded_file or uploaded_file.name != file_name:
+                if not renamed_file or renamed_file.name != file_name:
                     raise DestinationConnectionError(f"Upload failed for file '{file_name}'")
                 # Log details about the uploaded file
-                logger.info(f"Uploaded file {uploaded_file.name} with ID {uploaded_file.id}")
+                logger.info(f"Uploaded file {renamed_file.name} with ID {renamed_file.id}")
             except Exception as e:
                 logger.error(f"Failed to upload file '{file_name}' using resumable upload: {e}")
                 raise DestinationConnectionError(

unstructured_ingest-0.4.7/unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, AsyncIterator
+
+from pydantic import Field
+
+from unstructured_ingest.error import (
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    FileData,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.onedrive import (
+    OnedriveAccessConfig,
+    OnedriveConnectionConfig,
+    OnedriveDownloader,
+    OnedriveDownloaderConfig,
+    OnedriveIndexer,
+    OnedriveIndexerConfig,
+)
+
+if TYPE_CHECKING:
+    from office365.onedrive.driveitems.driveItem import DriveItem
+
+CONNECTOR_TYPE = "sharepoint"
+
+
+class SharepointAccessConfig(OnedriveAccessConfig):
+    client_cred: str = Field(description="Microsoft App client secret")
+
+
+class SharepointConnectionConfig(OnedriveConnectionConfig):
+    site: str = Field(
+        description="Sharepoint site url. Process either base url e.g \
+                    https://[tenant].sharepoint.com  or relative sites \
+                    https://[tenant].sharepoint.com/sites/<site_name>. \
+                    To process all sites within the tenant pass a site url as \
+                    https://[tenant]-admin.sharepoint.com.\
+                    This requires the app to be registered at a tenant level"
+    )
+
+
+class SharepointIndexerConfig(OnedriveIndexerConfig):
+    pass
+
+
+@dataclass
+class SharepointIndexer(OnedriveIndexer):
+    connection_config: SharepointConnectionConfig
+    index_config: SharepointIndexerConfig
+
+    @requires_dependencies(["office365"], extras="sharepoint")
+    async def run_async(self, **kwargs: Any) -> AsyncIterator[FileData]:
+        from office365.runtime.client_request_exception import ClientRequestException
+
+        token_resp = await asyncio.to_thread(self.connection_config.get_token)
+        if "error" in token_resp:
+            raise SourceConnectionError(
+                f"[{CONNECTOR_TYPE}]: {token_resp['error']} ({token_resp.get('error_description')})"
+            )
+
+        client = await asyncio.to_thread(self.connection_config.get_client)
+        try:
+            site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
+            site_drive_item = site.drive.get().execute_query().root
+        except ClientRequestException:
+            logger.info("Site not found")
+
+        drive_items = await self.list_objects(
+            folder=site_drive_item, recursive=self.index_config.recursive
+        )
+        for drive_item in drive_items:
+            file_data = await self.drive_item_to_file_data(drive_item=drive_item)
+            yield file_data
+
+
+class SharepointDownloaderConfig(OnedriveDownloaderConfig):
+    pass
+
+
+@dataclass
+class SharepointDownloader(OnedriveDownloader):
+    connection_config: SharepointConnectionConfig
+    download_config: SharepointDownloaderConfig
+
+    @SourceConnectionNetworkError.wrap
+    @requires_dependencies(["office365"], extras="onedrive")
+    def _fetch_file(self, file_data: FileData) -> DriveItem:
+        from office365.runtime.client_request_exception import ClientRequestException
+
+        if file_data.source_identifiers is None or not file_data.source_identifiers.fullpath:
+            raise ValueError(
+                f"file data doesn't have enough information to get "
+                f"file content: {file_data.model_dump()}"
+            )
+
+        server_relative_path = file_data.source_identifiers.fullpath
+        client = self.connection_config.get_client()
+
+        try:
+            site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
+            site_drive_item = site.drive.get().execute_query().root
+        except ClientRequestException:
+            logger.info("Site not found")
+        file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
+
+        if not file:
+            raise FileNotFoundError(f"file not found: {server_relative_path}")
+        return file
+
+
+sharepoint_source_entry = SourceRegistryEntry(
+    connection_config=SharepointConnectionConfig,
+    indexer_config=SharepointIndexerConfig,
+    indexer=SharepointIndexer,
+    downloader_config=SharepointDownloaderConfig,
+    downloader=SharepointDownloader,
+)

{unstructured_ingest-0.4.5 → unstructured_ingest-0.4.7}/unstructured_ingest/v2/processes/connectors/sql/databricks_delta_tables.py

@@ -51,9 +51,10 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
 
         host = f"https://{self.server_hostname}"
         access_configs = self.access_config.get_secret_value()
-        if (client_id := access_configs.client_id) and (
-            client_secret := access_configs.client_secret
-        ):
+        client_id = access_configs.client_id
+        client_secret = access_configs.client_secret
+
+        def _get_credentials_provider():
             return oauth_service_principal(
                 Config(
                     host=host,
@@ -61,6 +62,10 @@ class DatabricksDeltaTablesConnectionConfig(SQLConnectionConfig):
                     client_secret=client_secret,
                 )
             )
+
+        if client_id and client_secret:
+            return _get_credentials_provider
+
         return False
 
     def model_post_init(self, __context: Any) -> None: