unstructured-ingest 0.5.9__py3-none-any.whl → 0.5.11__py3-none-any.whl

test/unit/v2/connectors/test_jira.py

@@ -0,0 +1,401 @@
+from unittest.mock import MagicMock
+
+import pytest
+from pydantic import ValidationError
+from pytest_mock import MockerFixture
+
+from unstructured_ingest.v2.processes.connectors.jira import (
+    FieldGetter,
+    JiraAccessConfig,
+    JiraConnectionConfig,
+    JiraIndexer,
+    JiraIndexerConfig,
+    JiraIssueMetadata,
+    issues_fetcher_wrapper,
+    nested_object_to_field_getter,
+)
+
+
+@pytest.fixture
+def jira_connection_config():
+    access_config = JiraAccessConfig(password="password")
+    return JiraConnectionConfig(
+        url="http://localhost:1234",
+        username="test@example.com",
+        access_config=access_config,
+    )
+
+
+@pytest.fixture
+def jira_indexer(jira_connection_config: JiraConnectionConfig):
+    indexer_config = JiraIndexerConfig(projects=["TEST1"], boards=["2"], issues=["TEST2-1"])
+    return JiraIndexer(connection_config=jira_connection_config, index_config=indexer_config)
+
+
+@pytest.fixture
+def mock_jira(mocker: MockerFixture):
+    mock_client = mocker.patch.object(JiraConnectionConfig, "get_client", autospec=True)
+    mock_jira = mocker.MagicMock()
+    mock_client.return_value.__enter__.return_value = mock_jira
+    return mock_jira
+
+
+def test_jira_indexer_precheck_success(
+    caplog: pytest.LogCaptureFixture,
+    mocker: MockerFixture,
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    get_permissions = mocker.MagicMock()
+    get_permissions.return_value = {"permissions": {"BROWSE_PROJECTS": {"havePermission": True}}}
+    mock_jira.get_permissions = get_permissions
+
+    with caplog.at_level("INFO"):
+        jira_indexer.precheck()
+        assert "Connection to Jira successful." in caplog.text
+
+    get_permissions.assert_called_once()
+
+
+def test_jira_indexer_precheck_no_permission(
+    mocker: MockerFixture,
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    get_permissions = mocker.MagicMock()
+    get_permissions.return_value = {"permissions": {"BROWSE_PROJECTS": {"havePermission": False}}}
+    mock_jira.get_permissions = get_permissions
+
+    with pytest.raises(ValueError):
+        jira_indexer.precheck()
+
+    get_permissions.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    ("project_issues_count", "expected_issues_count"), [(2, 2), ({"total": 2}, 2), (0, 0)]
+)
+def test_jira_indexer_get_issues_within_single_project(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+    project_issues_count,
+    expected_issues_count,
+):
+    mock_jira.get_project_issues_count.return_value = project_issues_count
+    mock_jira.get_all_project_issues.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_single_project("TEST1")
+    assert len(issues) == expected_issues_count
+
+    if issues:
+        assert issues[0].id == "1"
+        assert issues[0].key == "TEST-1"
+        assert issues[1].id == "2"
+        assert issues[1].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_single_project_error(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_project_issues_count.return_value = {}
+
+    with pytest.raises(KeyError):
+        jira_indexer._get_issues_within_single_project("TEST1")
+
+
+def test_jira_indexer_get_issues_within_projects_with_projects(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_project_issues_count.return_value = 2
+    mock_jira.get_all_project_issues.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_projects()
+    assert len(issues) == 2
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_projects_no_projects_with_boards_or_issues(
+    mocker: MockerFixture,
+    jira_indexer: JiraIndexer,
+):
+    jira_indexer.index_config.projects = None
+    jira_indexer.index_config.boards = ["2"]
+    mocker.patch.object(JiraConnectionConfig, "get_client", autospec=True)
+
+    issues = jira_indexer._get_issues_within_projects()
+    assert issues == []
+
+
+def test_jira_indexer_get_issues_within_projects_no_projects_no_boards_no_issues(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    jira_indexer.index_config.projects = None
+    jira_indexer.index_config.boards = None
+    jira_indexer.index_config.issues = None
+    mock_jira.projects.return_value = [{"key": "TEST1"}, {"key": "TEST2"}]
+    mock_jira.get_project_issues_count.return_value = 2
+    mock_jira.get_all_project_issues.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_projects()
+    assert len(issues) == 4
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[2].id == "1"
+    assert issues[2].key == "TEST-1"
+    assert issues[3].id == "2"
+    assert issues[3].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_boards(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_issues_for_board.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_boards()
+    assert len(issues) == 2
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_single_board(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_issues_for_board.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_single_board("1")
+    assert len(issues) == 2
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[0].board_id == "1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[1].board_id == "1"
+
+
+def test_jira_indexer_get_issues_within_single_board_no_issues(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_issues_for_board.return_value = []
+
+    issues = jira_indexer._get_issues_within_single_board("1")
+    assert len(issues) == 0
+
+
+def test_jira_indexer_get_issues(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    jira_indexer.index_config.issues = ["TEST2-1", "TEST2-2"]
+    mock_jira.get_issue.return_value = {
+        "id": "ISSUE_ID",
+        "key": "ISSUE_KEY",
+    }
+
+    issues = jira_indexer._get_issues()
+    assert len(issues) == 2
+    assert issues[0].id == "ISSUE_ID"
+    assert issues[0].key == "ISSUE_KEY"
+
+
+def test_jira_indexer_get_issues_unique_issues(mocker: MockerFixture, jira_indexer: JiraIndexer):
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_boards",
+        return_value=[
+            JiraIssueMetadata(id="1", key="TEST-1", board_id="1"),
+            JiraIssueMetadata(id="2", key="TEST-2", board_id="1"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_projects",
+        return_value=[
+            JiraIssueMetadata(id="1", key="TEST-1"),
+            JiraIssueMetadata(id="3", key="TEST-3"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues",
+        return_value=[
+            JiraIssueMetadata(id="4", key="TEST-4"),
+            JiraIssueMetadata(id="2", key="TEST-2"),
+        ],
+    )
+
+    issues = jira_indexer.get_issues()
+    assert len(issues) == 4
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[0].board_id == "1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[1].board_id == "1"
+    assert issues[2].id == "3"
+    assert issues[2].key == "TEST-3"
+    assert issues[3].id == "4"
+    assert issues[3].key == "TEST-4"
+
+
+def test_jira_indexer_get_issues_no_duplicates(mocker: MockerFixture, jira_indexer: JiraIndexer):
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_boards",
+        return_value=[
+            JiraIssueMetadata(id="1", key="TEST-1", board_id="1"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_projects",
+        return_value=[
+            JiraIssueMetadata(id="2", key="TEST-2"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues",
+        return_value=[
+            JiraIssueMetadata(id="3", key="TEST-3"),
+        ],
+    )
+
+    issues = jira_indexer.get_issues()
+    assert len(issues) == 3
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[0].board_id == "1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[2].id == "3"
+    assert issues[2].key == "TEST-3"
+
+
+def test_jira_indexer_get_issues_empty(mocker: MockerFixture, jira_indexer: JiraIndexer):
+    mocker.patch.object(JiraIndexer, "_get_issues_within_boards", return_value=[])
+    mocker.patch.object(JiraIndexer, "_get_issues_within_projects", return_value=[])
+    mocker.patch.object(JiraIndexer, "_get_issues", return_value=[])
+
+    issues = jira_indexer.get_issues()
+    assert len(issues) == 0
+
+
+def test_connection_config_multiple_auth():
+    with pytest.raises(ValidationError):
+        JiraConnectionConfig(
+            access_config=JiraAccessConfig(
+                password="api_token",
+                token="access_token",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
+def test_connection_config_no_auth():
+    with pytest.raises(ValidationError):
+        JiraConnectionConfig(access_config=JiraAccessConfig(), url="url")
+
+
+def test_connection_config_basic_auth():
+    JiraConnectionConfig(
+        access_config=JiraAccessConfig(password="api_token"),
+        url="url",
+        username="user_email",
+    )
+
+
+def test_connection_config_pat_auth():
+    JiraConnectionConfig(
+        access_config=JiraAccessConfig(token="access_token"),
+        url="url",
+    )
+
+
+def test_jira_issue_metadata_object():
+    expected = {"id": "10000", "key": "TEST-1", "board_id": "1", "project_id": "TEST"}
+    metadata = JiraIssueMetadata(id="10000", key="TEST-1", board_id="1")
+    assert expected == metadata.to_dict()
+
+
+def test_nested_object_to_field_getter():
+    obj = {"a": 1, "b": {"c": 2}}
+    fg = nested_object_to_field_getter(obj)
+    assert isinstance(fg, FieldGetter)
+    assert fg["a"] == 1
+    assert isinstance(fg["b"], FieldGetter)
+    assert fg["b"]["c"] == 2
+    assert isinstance(fg["b"]["d"], FieldGetter)
+    assert fg["b"]["d"]["e"] == {}
+
+
+def test_issues_fetcher_wrapper():
+    test_issues_to_fetch = 250
+    test_issues = [{"id": i} for i in range(0, test_issues_to_fetch)]
+
+    def mock_func(limit, start):
+        return {"results": test_issues[start : start + limit]}
+
+    wrapped_func = issues_fetcher_wrapper(mock_func, number_of_issues_to_fetch=test_issues_to_fetch)
+    results = wrapped_func()
+    assert len(results) == 250
+    assert results[0]["id"] == 0
+    assert results[-1]["id"] == 249
+
+    test_issues_to_fetch = 150
+    test_issues = [{"id": i} for i in range(0, test_issues_to_fetch)]
+
+    def mock_func_list(limit, start):
+        return test_issues[start : start + limit]
+
+    wrapped_func_list = issues_fetcher_wrapper(
+        mock_func_list, number_of_issues_to_fetch=test_issues_to_fetch
+    )
+    results_list = wrapped_func_list()
+    assert len(results_list) == 150
+    assert results_list[0]["id"] == 0
+    assert results_list[-1]["id"] == 149
+
+    def mock_func_invalid(limit, start):
+        return "invalid"
+
+    wrapped_func_invalid = issues_fetcher_wrapper(mock_func_invalid, number_of_issues_to_fetch=50)
+    with pytest.raises(TypeError):
+        wrapped_func_invalid()
+
+    def mock_func_key_error(limit, start):
+        return {"wrong_key": []}
+
+    wrapped_func_key_error = issues_fetcher_wrapper(
+        mock_func_key_error, number_of_issues_to_fetch=50
+    )
+    with pytest.raises(KeyError):
+        wrapped_func_key_error()

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.9"  # pragma: no cover
+__version__ = "0.5.11"  # pragma: no cover

unstructured_ingest/embed/openai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field, SecretStr
 
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
 class OpenAIEmbeddingConfig(EmbeddingConfig):
     api_key: SecretStr
     embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
+    base_url: Optional[str] = None
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
@@ -57,13 +58,13 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
     def get_client(self) -> "OpenAI":
         from openai import OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value())
+        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
 
     @requires_dependencies(["openai"], extras="openai")
     def get_async_client(self) -> "AsyncOpenAI":
         from openai import AsyncOpenAI
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value())
+        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
 
 
 @dataclass

unstructured_ingest/utils/string_and_date_utils.py

@@ -1,9 +1,12 @@
 import json
+import re
 import typing as t
 from datetime import datetime
 
 from dateutil import parser
 
+from unstructured_ingest.v2.logger import logger
+
 
 def json_to_dict(json_string: str) -> t.Union[str, t.Dict[str, t.Any]]:
     """Helper function attempts to deserialize json string to a dictionary."""
@@ -47,3 +50,25 @@ def truncate_string_bytes(string: str, max_bytes: int, encoding: str = "utf-8")
     if len(encoded_string) <= max_bytes:
         return string
     return encoded_string[:max_bytes].decode(encoding, errors="ignore")
+
+
+def fix_unescaped_unicode(text: str, encoding: str = "utf-8") -> str:
+    """
+    Fix unescaped Unicode sequences in text.
+    """
+    try:
+        _text: str = json.dumps(text)
+
+        # Pattern to match unescaped Unicode sequences like \\uXXXX
+        pattern = r"\\\\u([0-9A-Fa-f]{4})"
+        # Replace with properly escaped Unicode sequences \uXXXX
+        _text = re.sub(pattern, r"\\u\1", _text)
+        _text = json.loads(_text)
+
+        # Encode the text to check for encoding errors
+        _text.encode(encoding)
+        return _text
+    except Exception as e:
+        # Return original text if encoding fails
+        logger.warning(f"Failed to fix unescaped Unicode sequences: {e}", exc_info=True)
+        return text

unstructured_ingest/v2/interfaces/downloader.py

@@ -1,5 +1,5 @@
 import os
-from abc import ABC, abstractmethod
+from abc import ABC
 from pathlib import Path
 from typing import Any, Optional, TypedDict, TypeVar, Union
 
@@ -81,9 +81,8 @@ class Downloader(BaseProcess, BaseConnector, ABC):
     def is_async(self) -> bool:
         return True
 
-    @abstractmethod
     def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
-        pass
+        raise NotImplementedError()
 
     async def run_async(self, file_data: FileData, **kwargs: Any) -> download_responses:
         return self.run(file_data=file_data, **kwargs)

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -34,6 +34,8 @@ from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
 from .google_drive import google_drive_source_entry
+from .jira import CONNECTOR_TYPE as JIRA_CONNECTOR_TYPE
+from .jira import jira_source_entry
 from .kdbai import CONNECTOR_TYPE as KDBAI_CONNECTOR_TYPE
 from .kdbai import kdbai_destination_entry
 from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
@@ -115,3 +117,5 @@ add_source_entry(source_type=CONFLUENCE_CONNECTOR_TYPE, entry=confluence_source_
 
 add_source_entry(source_type=DISCORD_CONNECTOR_TYPE, entry=discord_source_entry)
 add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destination_entry)
+
+add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)

unstructured_ingest/v2/processes/connectors/astradb.py

@@ -144,10 +144,6 @@ async def get_async_astra_collection(
     return async_astra_db_collection
 
 
-class AstraDBUploadStagerConfig(UploadStagerConfig):
-    pass
-
-
 class AstraDBIndexerConfig(IndexerConfig):
     collection_name: str = Field(
         description="The name of the Astra DB collection. "
@@ -158,30 +154,6 @@ class AstraDBIndexerConfig(IndexerConfig):
     batch_size: int = Field(default=20, description="Number of records per batch")
 
 
-class AstraDBDownloaderConfig(DownloaderConfig):
-    fields: list[str] = field(default_factory=list)
-
-
-class AstraDBUploaderConfig(UploaderConfig):
-    collection_name: Optional[str] = Field(
-        description="The name of the Astra DB collection. "
-        "Note that the collection name must only include letters, "
-        "numbers, and underscores.",
-        default=None,
-    )
-    keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
-    requested_indexing_policy: Optional[dict[str, Any]] = Field(
-        default=None,
-        description="The indexing policy to use for the collection.",
-        examples=['{"deny": ["metadata"]}'],
-    )
-    batch_size: int = Field(default=20, description="Number of records per batch")
-    record_id_key: str = Field(
-        default=RECORD_ID_LABEL,
-        description="searchable key to find entries for the same record on previous runs",
-    )
-
-
 @dataclass
 class AstraDBIndexer(Indexer):
     connection_config: AstraDBConnectionConfig
@@ -239,6 +211,10 @@ class AstraDBIndexer(Indexer):
             yield fd
 
 
+class AstraDBDownloaderConfig(DownloaderConfig):
+    fields: list[str] = field(default_factory=list)
+
+
 @dataclass
 class AstraDBDownloader(Downloader):
     connection_config: AstraDBConnectionConfig
@@ -315,6 +291,12 @@ class AstraDBDownloader(Downloader):
         return download_responses
 
 
+class AstraDBUploadStagerConfig(UploadStagerConfig):
+    flatten_metadata: Optional[bool] = Field(
+        default=False, description="Move metadata to top level of the record."
+    )
+
+
 @dataclass
 class AstraDBUploadStager(UploadStager):
     upload_stager_config: AstraDBUploadStagerConfig = field(
@@ -336,6 +318,12 @@ class AstraDBUploadStager(UploadStager):
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         self.truncate_dict_elements(element_dict)
+        if self.upload_stager_config.flatten_metadata:
+            # move metadata to top level so it isn't nested in metadata column
+            metadata = element_dict.pop("metadata", None)
+            if metadata:
+                element_dict.update(metadata)
+
         return {
             "$vector": element_dict.pop("embeddings", None),
             "content": element_dict.pop("text", None),
@@ -344,6 +332,26 @@ class AstraDBUploadStager(UploadStager):
         }
 
 
+class AstraDBUploaderConfig(UploaderConfig):
+    collection_name: Optional[str] = Field(
+        description="The name of the Astra DB collection. "
+        "Note that the collection name must only include letters, "
+        "numbers, and underscores.",
+        default=None,
+    )
+    keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
+    requested_indexing_policy: Optional[dict[str, Any]] = Field(
+        default=None,
+        description="The indexing policy to use for the collection.",
+        examples=['{"deny": ["metadata"]}'],
+    )
+    batch_size: int = Field(default=20, description="Number of records per batch")
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="searchable key to find entries for the same record on previous runs",
+    )
+
+
 @dataclass
 class AstraDBUploader(Uploader):
     connection_config: AstraDBConnectionConfig

unstructured_ingest/v2/processes/connectors/confluence.py

@@ -8,6 +8,7 @@ from pydantic import Field, Secret
 from unstructured_ingest.error import SourceConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.html import HtmlMixin
+from unstructured_ingest.utils.string_and_date_utils import fix_unescaped_unicode
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -224,7 +225,6 @@ class ConfluenceDownloader(Downloader):
                     page_id=doc_id,
                     expand="history.lastUpdated,version,body.view",
                 )
-
         except Exception as e:
             logger.error(f"Failed to retrieve page with ID {doc_id}: {e}", exc_info=True)
             raise SourceConnectionError(f"Failed to retrieve page with ID {doc_id}: {e}")
@@ -236,7 +236,7 @@ class ConfluenceDownloader(Downloader):
         title = page["title"]
         # Using h1 for title is supported by both v1 and v2 html parsing in unstructured
         title_html = f"<h1>{title}</h1>"
-        content = f"<body class='Document' >{title_html}{content}</body>"
+        content = fix_unescaped_unicode(f"<body class='Document' >{title_html}{content}</body>")
         if self.download_config.extract_images:
             with self.connection_config.get_client() as client:
                 content = self.download_config.extract_html_images(

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -92,6 +92,7 @@ class DeltaTableUploadStager(UploadStager):
         output_path = Path(output_dir) / Path(f"{output_filename}.parquet")
 
         df = convert_to_pandas_dataframe(elements_dict=elements_contents)
+        df = df.dropna(axis=1, how="all")
         df.to_parquet(output_path)
 
         return output_path
@@ -153,6 +154,7 @@ class DeltaTableUploader(Uploader):
             "table_or_uri": updated_upload_path,
             "data": df,
             "mode": "overwrite",
+            "schema_mode": "merge",
             "storage_options": storage_options,
         }
         queue = Queue()