unstructured-ingest 0.5.9__py3-none-any.whl → 0.5.11__py3-none-any.whl

test/integration/connectors/test_astradb.py

@@ -31,6 +31,7 @@ from unstructured_ingest.v2.processes.connectors.astradb import (
     AstraDBUploader,
     AstraDBUploaderConfig,
     AstraDBUploadStager,
+    AstraDBUploadStagerConfig,
     DestinationConnectionError,
     SourceConnectionError,
 )
@@ -258,3 +259,23 @@ def test_astra_stager(
         stager=stager,
         tmp_dir=tmp_path,
     )
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_astra_stager_flatten_metadata(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    stager_config = AstraDBUploadStagerConfig(flatten_metadata=True)
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = AstraDBUploadStager(upload_stager_config=stager_config)
+    stager_validation(
+        configs=StagerValidationConfigs(
+            test_id=CONNECTOR_TYPE, expected_count=22, expected_folder="stager_flatten_metadata"
+        ),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

test/integration/connectors/test_dropbox.py

@@ -0,0 +1,151 @@
+import os
+
+import pytest
+import requests
+
+from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
+    CONNECTOR_TYPE as DROPBOX_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
+    DropboxAccessConfig,
+    DropboxConnectionConfig,
+    DropboxDownloader,
+    DropboxDownloaderConfig,
+    DropboxIndexer,
+    DropboxIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
+async def test_dropbox_source(temp_dir):
+    """
+    Integration test for the Dropbox source connector.
+
+    This test indexes data from dropbox://test-input/ and downloads the resulting files,
+    then compares them to fixture data.
+    """
+    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
+    app_key = os.getenv("DROPBOX_APP_KEY")
+    app_secret = os.getenv("DROPBOX_APP_SECRET")
+
+    connection_config = DropboxConnectionConfig(
+        access_config=DropboxAccessConfig(
+            refresh_token=refresh_token,
+            app_key=app_key,
+            app_secret=app_secret,
+        )
+    )
+
+    index_config = DropboxIndexerConfig(
+        recursive=True,
+        remote_url="dropbox://test-input",
+    )
+    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
+
+    indexer = DropboxIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = DropboxDownloader(
+        connection_config=connection_config,
+        download_config=downloader_config,
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="dropbox",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+            ],
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
+async def test_dropbox_short_lived_token_via_refresh(temp_dir):
+    """
+    Demonstrates manually generating an access token from refresh credentials,
+    then passing ONLY the short-lived token to the Dropbox connector
+    (no app_key, app_secret, or refresh_token in the actual connection config).
+
+    This effectively mimics an external system that hands us a short-lived token.
+    """
+    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
+    app_key = os.getenv("DROPBOX_APP_KEY")
+    app_secret = os.getenv("DROPBOX_APP_SECRET")
+
+    #    Manually request a short-lived token from Dropbox's OAuth endpoint
+    #    This call is basically what the connector code does internally,
+    #    but we're doing it here in the test so we can pass only the short-lived token later.
+    response = requests.post(
+        "https://api.dropboxapi.com/oauth2/token",
+        data={
+            "grant_type": "refresh_token",
+            "refresh_token": refresh_token,
+        },
+        auth=(app_key, app_secret),
+        timeout=30,  # seconds
+    )
+    response.raise_for_status()
+    data = response.json()
+    short_lived_token = data["access_token"]
+    print("Acquired an access token from Dropbox")
+
+    #    Build connection config with ONLY the short-lived token
+    #    We omit refresh_token, app_key, and app_secret to confirm that
+    #    our connector can operate purely on the short-lived token.
+    connection_config = DropboxConnectionConfig(
+        access_config=DropboxAccessConfig(
+            token=short_lived_token,
+            app_key=None,
+            app_secret=None,
+            refresh_token=None,
+        )
+    )
+
+    index_config = DropboxIndexerConfig(
+        recursive=True,
+        remote_url="dropbox://test-input",
+    )
+    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
+
+    indexer = DropboxIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = DropboxDownloader(
+        connection_config=connection_config,
+        download_config=downloader_config,
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="dropbox_short_lived_via_refresh",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+            ],
+        ),
+    )

test/integration/connectors/test_jira.py

@@ -0,0 +1,67 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.jira import (
+    CONNECTOR_TYPE,
+    JiraAccessConfig,
+    JiraConnectionConfig,
+    JiraDownloader,
+    JiraDownloaderConfig,
+    JiraIndexer,
+    JiraIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
+@requires_env("JIRA_INGEST_USER_EMAIL", "JIRA_INGEST_API_TOKEN")
+async def test_jira_source(temp_dir):
+    # Retrieve environment variables
+    jira_url = os.environ.get(
+        "JIRA_INGEST_URL", "https://unstructured-jira-connector-test.atlassian.net"
+    )
+    user_email = os.environ["JIRA_INGEST_USER_EMAIL"]
+    api_token = os.environ["JIRA_INGEST_API_TOKEN"]
+    projects = ["JCTP1"]
+    boards = ["3"]
+    issues = ["JCTP2-1", "JCTP2-2", "JCTP2-3"]
+
+    # Create connection and indexer configurations
+    access_config = JiraAccessConfig(password=api_token)
+    connection_config = JiraConnectionConfig(
+        url=jira_url,
+        username=user_email,
+        access_config=access_config,
+    )
+    index_config = JiraIndexerConfig(projects=projects, boards=boards, issues=issues)
+
+    download_config = JiraDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = JiraIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = JiraDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="jira",
+            expected_num_files=8,
+            validate_file_data=True,
+            validate_downloaded_files=True,
+        ),
+    )

test/integration/connectors/test_zendesk.py

@@ -0,0 +1,142 @@
+import os
+from pathlib import Path
+from typing import Optional
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.errors import UserAuthError
+from unstructured_ingest.v2.processes.connectors.zendesk import (
+    CONNECTOR_TYPE,
+    ZendeskAccessConfig,
+    ZendeskConnectionConfig,
+    ZendeskDownloader,
+    ZendeskDownloaderConfig,
+    ZendeskIndexer,
+    ZendeskIndexerConfig,
+)
+
+
+async def zendesk_source_test(
+    tmp_path: Path,
+    token: Optional[str] = None,
+    email: Optional[str] = None,
+    subdomain: Optional[str] = None,
+):
+
+    access_config = ZendeskAccessConfig(api_token=token)
+    connection_config = ZendeskConnectionConfig(
+        subdomain=subdomain, email=email, access_config=access_config
+    )
+
+    index_config = ZendeskIndexerConfig(batch_size=2, item_type="tickets")
+
+    indexer = ZendeskIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # handle downloader.
+    download_config = ZendeskDownloaderConfig(download_dir=tmp_path)
+
+    downloader = ZendeskDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="zendesk-tickets",
+            expected_num_files=4,
+            validate_file_data=False,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+async def zendesk_source_articles_test(
+    tmp_path: Path,
+    token: Optional[str] = None,
+    email: Optional[str] = None,
+    subdomain: Optional[str] = None,
+):
+
+    access_config = ZendeskAccessConfig(api_token=token)
+    connection_config = ZendeskConnectionConfig(
+        subdomain=subdomain, email=email, access_config=access_config
+    )
+
+    index_config = ZendeskIndexerConfig(batch_size=2, item_type="articles")
+
+    indexer = ZendeskIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # handle downloader.
+    download_config = ZendeskDownloaderConfig(download_dir=tmp_path, extract_images=True)
+
+    downloader = ZendeskDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="zendesk-articles",
+            expected_num_files=4,
+            validate_file_data=False,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+@requires_env("ZENDESK_TOKEN")
+async def test_zendesk_source(temp_dir):
+    await zendesk_source_test(
+        tmp_path=temp_dir,
+        token=os.environ["ZENDESK_TOKEN"],
+        email="test@unstructured.io",
+        subdomain="unstructuredhelp",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+@requires_env("ZENDESK_TOKEN")
+async def test_zendesk_source_articles(temp_dir):
+    await zendesk_source_articles_test(
+        tmp_path=temp_dir,
+        token=os.environ["ZENDESK_TOKEN"],
+        email="test@unstructured.io",
+        subdomain="unstructuredhelp",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+async def test_zendesk_source_articles_fail(temp_dir):
+    with pytest.raises(expected_exception=UserAuthError):
+        await zendesk_source_articles_test(
+            tmp_path=temp_dir,
+            token="FORCE_FAIL_TOKEN",
+            email="test@unstructured.io",
+            subdomain="unstructuredhelp",
+        )

test/integration/connectors/utils/validation/destination.py

@@ -9,9 +9,10 @@ from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers, Uploa
 
 class StagerValidationConfigs(ValidationConfig):
     expected_count: int
+    expected_folder: str = "stager"
 
     def stager_output_dir(self) -> Path:
-        dir = self.test_output_dir() / "stager"
+        dir = self.test_output_dir() / self.expected_folder
         dir.mkdir(exist_ok=True, parents=True)
         return dir
 

test/unit/test_utils.py

@@ -10,6 +10,7 @@ from unstructured_ingest.cli.utils import extract_config
 from unstructured_ingest.interfaces import BaseConfig
 from unstructured_ingest.utils.string_and_date_utils import (
     ensure_isoformat_datetime,
+    fix_unescaped_unicode,
     json_to_dict,
     truncate_string_bytes,
 )
@@ -182,3 +183,29 @@ def test_truncate_string_bytes_return_untouched_string():
     result = truncate_string_bytes(test_string, max_bytes)
     assert result == "abcdef"
     assert len(result.encode("utf-8")) <= max_bytes
+
+
+def test_fix_unescaped_unicode_valid():
+    text = "This is a test with unescaped unicode: \\u0041"
+    expected = "This is a test with unescaped unicode: \u0041"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_no_unescaped_chars():
+    text = "This is a test with no unescaped unicode: \u0041"
+    expected = "This is a test with no unescaped unicode: \u0041"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_invalid_unicode():
+    text = "This is a test with invalid unescaped unicode: \\uZZZZ"
+    expected = "This is a test with invalid unescaped unicode: \\uZZZZ"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_encoding_error(caplog: pytest.LogCaptureFixture):
+    text = "This is a test with unescaped unicode: \\uD83D"
+    fix_unescaped_unicode(text)
+    with caplog.at_level("WARNING"):
+        fix_unescaped_unicode(text)
+        assert "Failed to fix unescaped Unicode sequences" in caplog.text