unstructured-ingest 0.5.9__tar.gz → 0.5.11__tar.gz

{unstructured_ingest-0.5.9/unstructured_ingest.egg-info → unstructured_ingest-0.5.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.9
+Version: 0.5.11
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,15 +22,15 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: tqdm
-Requires-Dist: pandas
 Requires-Dist: click
-Requires-Dist: dataclasses_json
 Requires-Dist: python-dateutil
-Requires-Dist: opentelemetry-sdk
 Requires-Dist: pydantic>=2.7
+Requires-Dist: tqdm
+Requires-Dist: dataclasses_json
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: pandas
 Provides-Extra: remote
-Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
+Requires-Dist: unstructured-client>=0.30.0; extra == "remote"
 Provides-Extra: csv
 Requires-Dist: unstructured[tsv]; extra == "csv"
 Provides-Extra: doc
@@ -66,16 +66,16 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -86,8 +86,8 @@ Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -125,13 +125,13 @@ Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
 Requires-Dist: neo4j-rust-ext; extra == "neo4j"
-Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -182,9 +182,9 @@ Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: ibis; extra == "vastdb"
-Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
+Requires-Dist: vastdb; extra == "vastdb"
+Requires-Dist: ibis; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
@@ -200,8 +200,8 @@ Provides-Extra: openai
 Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.9 → unstructured_ingest-0.5.11}/test/integration/connectors/test_astradb.py

@@ -31,6 +31,7 @@ from unstructured_ingest.v2.processes.connectors.astradb import (
     AstraDBUploader,
     AstraDBUploaderConfig,
     AstraDBUploadStager,
+    AstraDBUploadStagerConfig,
     DestinationConnectionError,
     SourceConnectionError,
 )
@@ -258,3 +259,23 @@ def test_astra_stager(
         stager=stager,
         tmp_dir=tmp_path,
     )
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_astra_stager_flatten_metadata(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    stager_config = AstraDBUploadStagerConfig(flatten_metadata=True)
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = AstraDBUploadStager(upload_stager_config=stager_config)
+    stager_validation(
+        configs=StagerValidationConfigs(
+            test_id=CONNECTOR_TYPE, expected_count=22, expected_folder="stager_flatten_metadata"
+        ),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

unstructured_ingest-0.5.11/test/integration/connectors/test_dropbox.py

@@ -0,0 +1,151 @@
+import os
+
+import pytest
+import requests
+
+from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
+    CONNECTOR_TYPE as DROPBOX_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
+    DropboxAccessConfig,
+    DropboxConnectionConfig,
+    DropboxDownloader,
+    DropboxDownloaderConfig,
+    DropboxIndexer,
+    DropboxIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
+async def test_dropbox_source(temp_dir):
+    """
+    Integration test for the Dropbox source connector.
+
+    This test indexes data from dropbox://test-input/ and downloads the resulting files,
+    then compares them to fixture data.
+    """
+    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
+    app_key = os.getenv("DROPBOX_APP_KEY")
+    app_secret = os.getenv("DROPBOX_APP_SECRET")
+
+    connection_config = DropboxConnectionConfig(
+        access_config=DropboxAccessConfig(
+            refresh_token=refresh_token,
+            app_key=app_key,
+            app_secret=app_secret,
+        )
+    )
+
+    index_config = DropboxIndexerConfig(
+        recursive=True,
+        remote_url="dropbox://test-input",
+    )
+    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
+
+    indexer = DropboxIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = DropboxDownloader(
+        connection_config=connection_config,
+        download_config=downloader_config,
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="dropbox",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+            ],
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(DROPBOX_CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("DROPBOX_REFRESH_TOKEN", "DROPBOX_APP_KEY", "DROPBOX_APP_SECRET")
+async def test_dropbox_short_lived_token_via_refresh(temp_dir):
+    """
+    Demonstrates manually generating an access token from refresh credentials,
+    then passing ONLY the short-lived token to the Dropbox connector
+    (no app_key, app_secret, or refresh_token in the actual connection config).
+
+    This effectively mimics an external system that hands us a short-lived token.
+    """
+    refresh_token = os.getenv("DROPBOX_REFRESH_TOKEN")
+    app_key = os.getenv("DROPBOX_APP_KEY")
+    app_secret = os.getenv("DROPBOX_APP_SECRET")
+
+    #    Manually request a short-lived token from Dropbox's OAuth endpoint
+    #    This call is basically what the connector code does internally,
+    #    but we're doing it here in the test so we can pass only the short-lived token later.
+    response = requests.post(
+        "https://api.dropboxapi.com/oauth2/token",
+        data={
+            "grant_type": "refresh_token",
+            "refresh_token": refresh_token,
+        },
+        auth=(app_key, app_secret),
+        timeout=30,  # seconds
+    )
+    response.raise_for_status()
+    data = response.json()
+    short_lived_token = data["access_token"]
+    print("Acquired an access token from Dropbox")
+
+    #    Build connection config with ONLY the short-lived token
+    #    We omit refresh_token, app_key, and app_secret to confirm that
+    #    our connector can operate purely on the short-lived token.
+    connection_config = DropboxConnectionConfig(
+        access_config=DropboxAccessConfig(
+            token=short_lived_token,
+            app_key=None,
+            app_secret=None,
+            refresh_token=None,
+        )
+    )
+
+    index_config = DropboxIndexerConfig(
+        recursive=True,
+        remote_url="dropbox://test-input",
+    )
+    downloader_config = DropboxDownloaderConfig(download_dir=temp_dir)
+
+    indexer = DropboxIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = DropboxDownloader(
+        connection_config=connection_config,
+        download_config=downloader_config,
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="dropbox_short_lived_via_refresh",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+            ],
+        ),
+    )

unstructured_ingest-0.5.11/test/integration/connectors/test_jira.py

@@ -0,0 +1,67 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.jira import (
+    CONNECTOR_TYPE,
+    JiraAccessConfig,
+    JiraConnectionConfig,
+    JiraDownloader,
+    JiraDownloaderConfig,
+    JiraIndexer,
+    JiraIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
+@requires_env("JIRA_INGEST_USER_EMAIL", "JIRA_INGEST_API_TOKEN")
+async def test_jira_source(temp_dir):
+    # Retrieve environment variables
+    jira_url = os.environ.get(
+        "JIRA_INGEST_URL", "https://unstructured-jira-connector-test.atlassian.net"
+    )
+    user_email = os.environ["JIRA_INGEST_USER_EMAIL"]
+    api_token = os.environ["JIRA_INGEST_API_TOKEN"]
+    projects = ["JCTP1"]
+    boards = ["3"]
+    issues = ["JCTP2-1", "JCTP2-2", "JCTP2-3"]
+
+    # Create connection and indexer configurations
+    access_config = JiraAccessConfig(password=api_token)
+    connection_config = JiraConnectionConfig(
+        url=jira_url,
+        username=user_email,
+        access_config=access_config,
+    )
+    index_config = JiraIndexerConfig(projects=projects, boards=boards, issues=issues)
+
+    download_config = JiraDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = JiraIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = JiraDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="jira",
+            expected_num_files=8,
+            validate_file_data=True,
+            validate_downloaded_files=True,
+        ),
+    )

unstructured_ingest-0.5.11/test/integration/connectors/test_zendesk.py

@@ -0,0 +1,142 @@
+import os
+from pathlib import Path
+from typing import Optional
+
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.errors import UserAuthError
+from unstructured_ingest.v2.processes.connectors.zendesk import (
+    CONNECTOR_TYPE,
+    ZendeskAccessConfig,
+    ZendeskConnectionConfig,
+    ZendeskDownloader,
+    ZendeskDownloaderConfig,
+    ZendeskIndexer,
+    ZendeskIndexerConfig,
+)
+
+
+async def zendesk_source_test(
+    tmp_path: Path,
+    token: Optional[str] = None,
+    email: Optional[str] = None,
+    subdomain: Optional[str] = None,
+):
+
+    access_config = ZendeskAccessConfig(api_token=token)
+    connection_config = ZendeskConnectionConfig(
+        subdomain=subdomain, email=email, access_config=access_config
+    )
+
+    index_config = ZendeskIndexerConfig(batch_size=2, item_type="tickets")
+
+    indexer = ZendeskIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # handle downloader.
+    download_config = ZendeskDownloaderConfig(download_dir=tmp_path)
+
+    downloader = ZendeskDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="zendesk-tickets",
+            expected_num_files=4,
+            validate_file_data=False,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+async def zendesk_source_articles_test(
+    tmp_path: Path,
+    token: Optional[str] = None,
+    email: Optional[str] = None,
+    subdomain: Optional[str] = None,
+):
+
+    access_config = ZendeskAccessConfig(api_token=token)
+    connection_config = ZendeskConnectionConfig(
+        subdomain=subdomain, email=email, access_config=access_config
+    )
+
+    index_config = ZendeskIndexerConfig(batch_size=2, item_type="articles")
+
+    indexer = ZendeskIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # handle downloader.
+    download_config = ZendeskDownloaderConfig(download_dir=tmp_path, extract_images=True)
+
+    downloader = ZendeskDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+        connector_type=CONNECTOR_TYPE,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="zendesk-articles",
+            expected_num_files=4,
+            validate_file_data=False,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+@requires_env("ZENDESK_TOKEN")
+async def test_zendesk_source(temp_dir):
+    await zendesk_source_test(
+        tmp_path=temp_dir,
+        token=os.environ["ZENDESK_TOKEN"],
+        email="test@unstructured.io",
+        subdomain="unstructuredhelp",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+@requires_env("ZENDESK_TOKEN")
+async def test_zendesk_source_articles(temp_dir):
+    await zendesk_source_articles_test(
+        tmp_path=temp_dir,
+        token=os.environ["ZENDESK_TOKEN"],
+        email="test@unstructured.io",
+        subdomain="unstructuredhelp",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+async def test_zendesk_source_articles_fail(temp_dir):
+    with pytest.raises(expected_exception=UserAuthError):
+        await zendesk_source_articles_test(
+            tmp_path=temp_dir,
+            token="FORCE_FAIL_TOKEN",
+            email="test@unstructured.io",
+            subdomain="unstructuredhelp",
+        )

{unstructured_ingest-0.5.9 → unstructured_ingest-0.5.11}/test/integration/connectors/utils/validation/destination.py

@@ -9,9 +9,10 @@ from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers, Uploa
 
 class StagerValidationConfigs(ValidationConfig):
     expected_count: int
+    expected_folder: str = "stager"
 
     def stager_output_dir(self) -> Path:
-        dir = self.test_output_dir() / "stager"
+        dir = self.test_output_dir() / self.expected_folder
         dir.mkdir(exist_ok=True, parents=True)
         return dir
 

{unstructured_ingest-0.5.9 → unstructured_ingest-0.5.11}/test/unit/test_utils.py

@@ -10,6 +10,7 @@ from unstructured_ingest.cli.utils import extract_config
 from unstructured_ingest.interfaces import BaseConfig
 from unstructured_ingest.utils.string_and_date_utils import (
     ensure_isoformat_datetime,
+    fix_unescaped_unicode,
     json_to_dict,
     truncate_string_bytes,
 )
@@ -182,3 +183,29 @@ def test_truncate_string_bytes_return_untouched_string():
     result = truncate_string_bytes(test_string, max_bytes)
     assert result == "abcdef"
     assert len(result.encode("utf-8")) <= max_bytes
+
+
+def test_fix_unescaped_unicode_valid():
+    text = "This is a test with unescaped unicode: \\u0041"
+    expected = "This is a test with unescaped unicode: \u0041"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_no_unescaped_chars():
+    text = "This is a test with no unescaped unicode: \u0041"
+    expected = "This is a test with no unescaped unicode: \u0041"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_invalid_unicode():
+    text = "This is a test with invalid unescaped unicode: \\uZZZZ"
+    expected = "This is a test with invalid unescaped unicode: \\uZZZZ"
+    assert fix_unescaped_unicode(text) == expected
+
+
+def test_fix_unescaped_unicode_encoding_error(caplog: pytest.LogCaptureFixture):
+    text = "This is a test with unescaped unicode: \\uD83D"
+    fix_unescaped_unicode(text)
+    with caplog.at_level("WARNING"):
+        fix_unescaped_unicode(text)
+        assert "Failed to fix unescaped Unicode sequences" in caplog.text