unstructured-ingest 0.5.8__py3-none-any.whl → 0.5.10__py3-none-any.whl

test/integration/connectors/test_astradb.py

@@ -56,12 +56,7 @@ def test_precheck_succeeds_indexer(connection_config: AstraDBConnectionConfig):
         connection_config=connection_config,
         index_config=AstraDBIndexerConfig(collection_name=EXISTENT_COLLECTION_NAME),
     )
-    uploader = AstraDBUploader(
-        connection_config=connection_config,
-        upload_config=AstraDBUploaderConfig(collection_name=EXISTENT_COLLECTION_NAME),
-    )
     indexer.precheck()
-    uploader.precheck()
 
 
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
@@ -73,6 +68,12 @@ def test_precheck_succeeds_uploader(connection_config: AstraDBConnectionConfig):
     )
     uploader.precheck()
 
+    uploader2 = AstraDBUploader(
+        connection_config=connection_config,
+        upload_config=AstraDBUploaderConfig(),
+    )
+    uploader2.precheck()
+
 
 @pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, VECTOR_DB_TAG)
 @requires_env("ASTRA_DB_APPLICATION_TOKEN", "ASTRA_DB_API_ENDPOINT")
@@ -216,6 +217,32 @@ async def test_astra_search_destination(
     )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+@requires_env("ASTRA_DB_API_ENDPOINT", "ASTRA_DB_APPLICATION_TOKEN")
+def test_astra_create_destination():
+    env_data = get_env_data()
+    connection_config = AstraDBConnectionConfig(
+        access_config=AstraDBAccessConfig(api_endpoint=env_data.api_endpoint, token=env_data.token),
+    )
+    uploader = AstraDBUploader(
+        connection_config=connection_config,
+        upload_config=AstraDBUploaderConfig(),
+    )
+    collection_name = "system_created-123"
+    formatted_collection_name = "system_created_123"
+    created = uploader.create_destination(destination_name=collection_name, vector_length=3072)
+    assert created
+    assert uploader.upload_config.collection_name == formatted_collection_name
+
+    created = uploader.create_destination(destination_name=collection_name, vector_length=3072)
+    assert not created
+
+    # cleanup
+    client = AstraDBClient()
+    db = client.get_database(api_endpoint=env_data.api_endpoint, token=env_data.token)
+    db.drop_collection(formatted_collection_name)
+
+
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_astra_stager(

test/integration/connectors/test_dropbox.py

@@ -0,0 +1,151 @@
+import os
+
+import pytest
+import requests
+
+from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
+    CONNECTOR_TYPE as DROPBOX_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
+    DropboxAccessConfig,
+    DropboxConnectionConfig,
+    DropboxDownloader,
+    DropboxDownloaderConfig,
+    DropboxIndexer,
+    DropboxIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
+async def test_dropbox_source(temp_dir):
+    """
+    Integration test for the Dropbox source connector.
+
+    This test indexes data from dropbox://test-input/ and downloads the resulting files,
+    then compares them to fixture data.
+    """
+    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
+    app_key = os.getenv("DROPBOX_APP_KEY")
+    app_secret = os.getenv("DROPBOX_APP_SECRET")
+
+    connection_config = DropboxConnectionConfig(
+        access_config=DropboxAccessConfig(
+            refresh_token=refresh_token,
+            app_key=app_key,
+            app_secret=app_secret,
+        )
+    )
+
+    index_config = DropboxIndexerConfig(
+        recursive=True,
+        remote_url="dropbox://test-input",
+    )
+    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
+
+    indexer = DropboxIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = DropboxDownloader(
+        connection_config=connection_config,
+        download_config=downloader_config,
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="dropbox",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+            ],
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
+async def test_dropbox_short_lived_token_via_refresh(temp_dir):
+    """
+    Demonstrates manually generating an access token from refresh credentials,
+    then passing ONLY the short-lived token to the Dropbox connector
+    (no app_key, app_secret, or refresh_token in the actual connection config).
+
+    This effectively mimics an external system that hands us a short-lived token.
+    """
+    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
+    app_key = os.getenv("DROPBOX_APP_KEY")
+    app_secret = os.getenv("DROPBOX_APP_SECRET")
+
+    #    Manually request a short-lived token from Dropbox's OAuth endpoint
+    #    This call is basically what the connector code does internally,
+    #    but we're doing it here in the test so we can pass only the short-lived token later.
+    response = requests.post(
+        "https://api.dropboxapi.com/oauth2/token",
+        data={
+            "grant_type": "refresh_token",
+            "refresh_token": refresh_token,
+        },
+        auth=(app_key, app_secret),
+        timeout=30,  # seconds
+    )
+    response.raise_for_status()
+    data = response.json()
+    short_lived_token = data["access_token"]
+    print("Acquired an access token from Dropbox")
+
+    #    Build connection config with ONLY the short-lived token
+    #    We omit refresh_token, app_key, and app_secret to confirm that
+    #    our connector can operate purely on the short-lived token.
+    connection_config = DropboxConnectionConfig(
+        access_config=DropboxAccessConfig(
+            token=short_lived_token,
+            app_key=None,
+            app_secret=None,
+            refresh_token=None,
+        )
+    )
+
+    index_config = DropboxIndexerConfig(
+        recursive=True,
+        remote_url="dropbox://test-input",
+    )
+    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
+
+    indexer = DropboxIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = DropboxDownloader(
+        connection_config=connection_config,
+        download_config=downloader_config,
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="dropbox_short_lived_via_refresh",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+            ],
+        ),
+    )

test/integration/connectors/test_jira.py

@@ -0,0 +1,67 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.jira import (
+    CONNECTOR_TYPE,
+    JiraAccessConfig,
+    JiraConnectionConfig,
+    JiraDownloader,
+    JiraDownloaderConfig,
+    JiraIndexer,
+    JiraIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
+@requires_env("JIRA_INGEST_USER_EMAIL", "JIRA_INGEST_API_TOKEN")
+async def test_jira_source(temp_dir):
+    # Retrieve environment variables
+    jira_url = os.environ.get(
+        "JIRA_INGEST_URL", "https://unstructured-jira-connector-test.atlassian.net"
+    )
+    user_email = os.environ["JIRA_INGEST_USER_EMAIL"]
+    api_token = os.environ["JIRA_INGEST_API_TOKEN"]
+    projects = ["JCTP1"]
+    boards = ["3"]
+    issues = ["JCTP2-1", "JCTP2-2", "JCTP2-3"]
+
+    # Create connection and indexer configurations
+    access_config = JiraAccessConfig(password=api_token)
+    connection_config = JiraConnectionConfig(
+        url=jira_url,
+        username=user_email,
+        access_config=access_config,
+    )
+    index_config = JiraIndexerConfig(projects=projects, boards=boards, issues=issues)
+
+    download_config = JiraDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = JiraIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = JiraDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="jira",
+            expected_num_files=8,
+            validate_file_data=True,
+            validate_downloaded_files=True,
+        ),
+    )

test/unit/test_utils.py

@@ -10,6 +10,7 @@ from unstructured_ingest.cli.utils import extract_config
 from unstructured_ingest.interfaces import BaseConfig
 from unstructured_ingest.utils.string_and_date_utils import (
     ensure_isoformat_datetime,
+    fix_unescaped_unicode,
     json_to_dict,
     truncate_string_bytes,
 )
@@ -182,3 +183,29 @@ def test_truncate_string_bytes_return_untouched_string():
     result = truncate_string_bytes(test_string, max_bytes)
     assert result == "abcdef"
     assert len(result.encode("utf-8")) <= max_bytes
+
+
+def test_fix_unescaped_unicode_valid():
+    text = "This is a test with unescaped unicode: \\u0041"
+    expected = "This is a test with unescaped unicode: \u0041"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_no_unescaped_chars():
+    text = "This is a test with no unescaped unicode: \u0041"
+    expected = "This is a test with no unescaped unicode: \u0041"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_invalid_unicode():
+    text = "This is a test with invalid unescaped unicode: \\uZZZZ"
+    expected = "This is a test with invalid unescaped unicode: \\uZZZZ"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_encoding_error(caplog: pytest.LogCaptureFixture):
+    text = "This is a test with unescaped unicode: \\uD83D"
+    fix_unescaped_unicode(text)
+    with caplog.at_level("WARNING"):
+        fix_unescaped_unicode(text)
+        assert "Failed to fix unescaped Unicode sequences" in caplog.text