unstructured-ingest 0.5.8__py3-none-any.whl → 0.5.10__py3-none-any.whl

test/unit/v2/connectors/test_jira.py

@@ -0,0 +1,401 @@
+from unittest.mock import MagicMock
+
+import pytest
+from pydantic import ValidationError
+from pytest_mock import MockerFixture
+
+from unstructured_ingest.v2.processes.connectors.jira import (
+    FieldGetter,
+    JiraAccessConfig,
+    JiraConnectionConfig,
+    JiraIndexer,
+    JiraIndexerConfig,
+    JiraIssueMetadata,
+    issues_fetcher_wrapper,
+    nested_object_to_field_getter,
+)
+
+
+@pytest.fixture
+def jira_connection_config():
+    access_config = JiraAccessConfig(password="password")
+    return JiraConnectionConfig(
+        url="http://localhost:1234",
+        username="test@example.com",
+        access_config=access_config,
+    )
+
+
+@pytest.fixture
+def jira_indexer(jira_connection_config: JiraConnectionConfig):
+    indexer_config = JiraIndexerConfig(projects=["TEST1"], boards=["2"], issues=["TEST2-1"])
+    return JiraIndexer(connection_config=jira_connection_config, index_config=indexer_config)
+
+
+@pytest.fixture
+def mock_jira(mocker: MockerFixture):
+    mock_client = mocker.patch.object(JiraConnectionConfig, "get_client", autospec=True)
+    mock_jira = mocker.MagicMock()
+    mock_client.return_value.__enter__.return_value = mock_jira
+    return mock_jira
+
+
+def test_jira_indexer_precheck_success(
+    caplog: pytest.LogCaptureFixture,
+    mocker: MockerFixture,
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    get_permissions = mocker.MagicMock()
+    get_permissions.return_value = {"permissions": {"BROWSE_PROJECTS": {"havePermission": True}}}
+    mock_jira.get_permissions = get_permissions
+
+    with caplog.at_level("INFO"):
+        jira_indexer.precheck()
+        assert "Connection to Jira successful." in caplog.text
+
+    get_permissions.assert_called_once()
+
+
+def test_jira_indexer_precheck_no_permission(
+    mocker: MockerFixture,
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    get_permissions = mocker.MagicMock()
+    get_permissions.return_value = {"permissions": {"BROWSE_PROJECTS": {"havePermission": False}}}
+    mock_jira.get_permissions = get_permissions
+
+    with pytest.raises(ValueError):
+        jira_indexer.precheck()
+
+    get_permissions.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    ("project_issues_count", "expected_issues_count"), [(2, 2), ({"total": 2}, 2), (0, 0)]
+)
+def test_jira_indexer_get_issues_within_single_project(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+    project_issues_count,
+    expected_issues_count,
+):
+    mock_jira.get_project_issues_count.return_value = project_issues_count
+    mock_jira.get_all_project_issues.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_single_project("TEST1")
+    assert len(issues) == expected_issues_count
+
+    if issues:
+        assert issues[0].id == "1"
+        assert issues[0].key == "TEST-1"
+        assert issues[1].id == "2"
+        assert issues[1].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_single_project_error(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_project_issues_count.return_value = {}
+
+    with pytest.raises(KeyError):
+        jira_indexer._get_issues_within_single_project("TEST1")
+
+
+def test_jira_indexer_get_issues_within_projects_with_projects(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_project_issues_count.return_value = 2
+    mock_jira.get_all_project_issues.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_projects()
+    assert len(issues) == 2
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_projects_no_projects_with_boards_or_issues(
+    mocker: MockerFixture,
+    jira_indexer: JiraIndexer,
+):
+    jira_indexer.index_config.projects = None
+    jira_indexer.index_config.boards = ["2"]
+    mocker.patch.object(JiraConnectionConfig, "get_client", autospec=True)
+
+    issues = jira_indexer._get_issues_within_projects()
+    assert issues == []
+
+
+def test_jira_indexer_get_issues_within_projects_no_projects_no_boards_no_issues(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    jira_indexer.index_config.projects = None
+    jira_indexer.index_config.boards = None
+    jira_indexer.index_config.issues = None
+    mock_jira.projects.return_value = [{"key": "TEST1"}, {"key": "TEST2"}]
+    mock_jira.get_project_issues_count.return_value = 2
+    mock_jira.get_all_project_issues.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_projects()
+    assert len(issues) == 4
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[2].id == "1"
+    assert issues[2].key == "TEST-1"
+    assert issues[3].id == "2"
+    assert issues[3].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_boards(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_issues_for_board.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_boards()
+    assert len(issues) == 2
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+
+
+def test_jira_indexer_get_issues_within_single_board(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_issues_for_board.return_value = [
+        {"id": "1", "key": "TEST-1"},
+        {"id": "2", "key": "TEST-2"},
+    ]
+
+    issues = jira_indexer._get_issues_within_single_board("1")
+    assert len(issues) == 2
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[0].board_id == "1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[1].board_id == "1"
+
+
+def test_jira_indexer_get_issues_within_single_board_no_issues(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    mock_jira.get_issues_for_board.return_value = []
+
+    issues = jira_indexer._get_issues_within_single_board("1")
+    assert len(issues) == 0
+
+
+def test_jira_indexer_get_issues(
+    jira_indexer: JiraIndexer,
+    mock_jira: MagicMock,
+):
+    jira_indexer.index_config.issues = ["TEST2-1", "TEST2-2"]
+    mock_jira.get_issue.return_value = {
+        "id": "ISSUE_ID",
+        "key": "ISSUE_KEY",
+    }
+
+    issues = jira_indexer._get_issues()
+    assert len(issues) == 2
+    assert issues[0].id == "ISSUE_ID"
+    assert issues[0].key == "ISSUE_KEY"
+
+
+def test_jira_indexer_get_issues_unique_issues(mocker: MockerFixture, jira_indexer: JiraIndexer):
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_boards",
+        return_value=[
+            JiraIssueMetadata(id="1", key="TEST-1", board_id="1"),
+            JiraIssueMetadata(id="2", key="TEST-2", board_id="1"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_projects",
+        return_value=[
+            JiraIssueMetadata(id="1", key="TEST-1"),
+            JiraIssueMetadata(id="3", key="TEST-3"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues",
+        return_value=[
+            JiraIssueMetadata(id="4", key="TEST-4"),
+            JiraIssueMetadata(id="2", key="TEST-2"),
+        ],
+    )
+
+    issues = jira_indexer.get_issues()
+    assert len(issues) == 4
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[0].board_id == "1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[1].board_id == "1"
+    assert issues[2].id == "3"
+    assert issues[2].key == "TEST-3"
+    assert issues[3].id == "4"
+    assert issues[3].key == "TEST-4"
+
+
+def test_jira_indexer_get_issues_no_duplicates(mocker: MockerFixture, jira_indexer: JiraIndexer):
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_boards",
+        return_value=[
+            JiraIssueMetadata(id="1", key="TEST-1", board_id="1"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues_within_projects",
+        return_value=[
+            JiraIssueMetadata(id="2", key="TEST-2"),
+        ],
+    )
+    mocker.patch.object(
+        JiraIndexer,
+        "_get_issues",
+        return_value=[
+            JiraIssueMetadata(id="3", key="TEST-3"),
+        ],
+    )
+
+    issues = jira_indexer.get_issues()
+    assert len(issues) == 3
+    assert issues[0].id == "1"
+    assert issues[0].key == "TEST-1"
+    assert issues[0].board_id == "1"
+    assert issues[1].id == "2"
+    assert issues[1].key == "TEST-2"
+    assert issues[2].id == "3"
+    assert issues[2].key == "TEST-3"
+
+
+def test_jira_indexer_get_issues_empty(mocker: MockerFixture, jira_indexer: JiraIndexer):
+    mocker.patch.object(JiraIndexer, "_get_issues_within_boards", return_value=[])
+    mocker.patch.object(JiraIndexer, "_get_issues_within_projects", return_value=[])
+    mocker.patch.object(JiraIndexer, "_get_issues", return_value=[])
+
+    issues = jira_indexer.get_issues()
+    assert len(issues) == 0
+
+
+def test_connection_config_multiple_auth():
+    with pytest.raises(ValidationError):
+        JiraConnectionConfig(
+            access_config=JiraAccessConfig(
+                password="api_token",
+                token="access_token",
+            ),
+            username="user_email",
+            url="url",
+        )
+
+
+def test_connection_config_no_auth():
+    with pytest.raises(ValidationError):
+        JiraConnectionConfig(access_config=JiraAccessConfig(), url="url")
+
+
+def test_connection_config_basic_auth():
+    JiraConnectionConfig(
+        access_config=JiraAccessConfig(password="api_token"),
+        url="url",
+        username="user_email",
+    )
+
+
+def test_connection_config_pat_auth():
+    JiraConnectionConfig(
+        access_config=JiraAccessConfig(token="access_token"),
+        url="url",
+    )
+
+
+def test_jira_issue_metadata_object():
+    expected = {"id": "10000", "key": "TEST-1", "board_id": "1", "project_id": "TEST"}
+    metadata = JiraIssueMetadata(id="10000", key="TEST-1", board_id="1")
+    assert expected == metadata.to_dict()
+
+
+def test_nested_object_to_field_getter():
+    obj = {"a": 1, "b": {"c": 2}}
+    fg = nested_object_to_field_getter(obj)
+    assert isinstance(fg, FieldGetter)
+    assert fg["a"] == 1
+    assert isinstance(fg["b"], FieldGetter)
+    assert fg["b"]["c"] == 2
+    assert isinstance(fg["b"]["d"], FieldGetter)
+    assert fg["b"]["d"]["e"] == {}
+
+
+def test_issues_fetcher_wrapper():
+    test_issues_to_fetch = 250
+    test_issues = [{"id": i} for i in range(0, test_issues_to_fetch)]
+
+    def mock_func(limit, start):
+        return {"results": test_issues[start : start + limit]}
+
+    wrapped_func = issues_fetcher_wrapper(mock_func, number_of_issues_to_fetch=test_issues_to_fetch)
+    results = wrapped_func()
+    assert len(results) == 250
+    assert results[0]["id"] == 0
+    assert results[-1]["id"] == 249
+
+    test_issues_to_fetch = 150
+    test_issues = [{"id": i} for i in range(0, test_issues_to_fetch)]
+
+    def mock_func_list(limit, start):
+        return test_issues[start : start + limit]
+
+    wrapped_func_list = issues_fetcher_wrapper(
+        mock_func_list, number_of_issues_to_fetch=test_issues_to_fetch
+    )
+    results_list = wrapped_func_list()
+    assert len(results_list) == 150
+    assert results_list[0]["id"] == 0
+    assert results_list[-1]["id"] == 149
+
+    def mock_func_invalid(limit, start):
+        return "invalid"
+
+    wrapped_func_invalid = issues_fetcher_wrapper(mock_func_invalid, number_of_issues_to_fetch=50)
+    with pytest.raises(TypeError):
+        wrapped_func_invalid()
+
+    def mock_func_key_error(limit, start):
+        return {"wrong_key": []}
+
+    wrapped_func_key_error = issues_fetcher_wrapper(
+        mock_func_key_error, number_of_issues_to_fetch=50
+    )
+    with pytest.raises(KeyError):
+        wrapped_func_key_error()

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.5.8"  # pragma: no cover
+__version__ = "0.5.10"  # pragma: no cover

unstructured_ingest/embed/openai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field, SecretStr
 
@@ -26,6 +26,7 @@ if TYPE_CHECKING:
 class OpenAIEmbeddingConfig(EmbeddingConfig):
     api_key: SecretStr
     embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
+    base_url: Optional[str] = None
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):
@@ -57,13 +58,13 @@ class OpenAIEmbeddingConfig(EmbeddingConfig):
     def get_client(self) -> "OpenAI":
         from openai import OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value())
+        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
 
     @requires_dependencies(["openai"], extras="openai")
     def get_async_client(self) -> "AsyncOpenAI":
         from openai import AsyncOpenAI
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value())
+        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
 
 
 @dataclass

unstructured_ingest/utils/string_and_date_utils.py

@@ -1,9 +1,12 @@
 import json
+import re
 import typing as t
 from datetime import datetime
 
 from dateutil import parser
 
+from unstructured_ingest.v2.logger import logger
+
 
 def json_to_dict(json_string: str) -> t.Union[str, t.Dict[str, t.Any]]:
     """Helper function attempts to deserialize json string to a dictionary."""
@@ -47,3 +50,25 @@ def truncate_string_bytes(string: str, max_bytes: int, encoding: str = "utf-8")
     if len(encoded_string) <= max_bytes:
         return string
     return encoded_string[:max_bytes].decode(encoding, errors="ignore")
+
+
+def fix_unescaped_unicode(text: str, encoding: str = "utf-8") -> str:
+    """
+    Fix unescaped Unicode sequences in text.
+    """
+    try:
+        _text: str = json.dumps(text)
+
+        # Pattern to match unescaped Unicode sequences like \\uXXXX
+        pattern = r"\\\\u([0-9A-Fa-f]{4})"
+        # Replace with properly escaped Unicode sequences \uXXXX
+        _text = re.sub(pattern, r"\\u\1", _text)
+        _text = json.loads(_text)
+
+        # Encode the text to check for encoding errors
+        _text.encode(encoding)
+        return _text
+    except Exception as e:
+        # Return original text if encoding fails
+        logger.warning(f"Failed to fix unescaped Unicode sequences: {e}", exc_info=True)
+        return text

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -34,6 +34,8 @@ from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
 from .google_drive import google_drive_source_entry
+from .jira import CONNECTOR_TYPE as JIRA_CONNECTOR_TYPE
+from .jira import jira_source_entry
 from .kdbai import CONNECTOR_TYPE as KDBAI_CONNECTOR_TYPE
 from .kdbai import kdbai_destination_entry
 from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
@@ -115,3 +117,5 @@ add_source_entry(source_type=CONFLUENCE_CONNECTOR_TYPE, entry=confluence_source_
 
 add_source_entry(source_type=DISCORD_CONNECTOR_TYPE, entry=discord_source_entry)
 add_destination_entry(destination_type=REDIS_CONNECTOR_TYPE, entry=redis_destination_entry)
+
+add_source_entry(source_type=JIRA_CONNECTOR_TYPE, entry=jira_source_entry)

unstructured_ingest/v2/processes/connectors/astradb.py

@@ -1,5 +1,6 @@
 import csv
 import hashlib
+import re
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
@@ -48,6 +49,7 @@ if TYPE_CHECKING:
     from astrapy import AsyncCollection as AstraDBAsyncCollection
     from astrapy import Collection as AstraDBCollection
     from astrapy import DataAPIClient as AstraDBClient
+    from astrapy import Database as AstraDB
 
 
 CONNECTOR_TYPE = "astradb"
@@ -85,11 +87,10 @@ class AstraDBConnectionConfig(ConnectionConfig):
         )
 
 
-def get_astra_collection(
+def get_astra_db(
     connection_config: AstraDBConnectionConfig,
-    collection_name: str,
     keyspace: str,
-) -> "AstraDBCollection":
+) -> "AstraDB":
     # Build the Astra DB object.
     access_configs = connection_config.access_config.get_secret_value()
 
@@ -103,9 +104,20 @@ def get_astra_collection(
         token=access_configs.token,
         keyspace=keyspace,
     )
+    return astra_db
+
 
-    # Connect to the collection
+def get_astra_collection(
+    connection_config: AstraDBConnectionConfig,
+    collection_name: str,
+    keyspace: str,
+) -> "AstraDBCollection":
+
+    astra_db = get_astra_db(connection_config=connection_config, keyspace=keyspace)
+
+    # astradb will return a collection object in all cases (even if it doesn't exist)
     astra_db_collection = astra_db.get_collection(name=collection_name)
+
     return astra_db_collection
 
 
@@ -151,10 +163,11 @@ class AstraDBDownloaderConfig(DownloaderConfig):
 
 
 class AstraDBUploaderConfig(UploaderConfig):
-    collection_name: str = Field(
+    collection_name: Optional[str] = Field(
         description="The name of the Astra DB collection. "
         "Note that the collection name must only include letters, "
-        "numbers, and underscores."
+        "numbers, and underscores.",
+        default=None,
     )
     keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
     requested_indexing_policy: Optional[dict[str, Any]] = Field(
@@ -337,25 +350,84 @@ class AstraDBUploader(Uploader):
     upload_config: AstraDBUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    def init(self, **kwargs: Any) -> None:
+        self.create_destination(**kwargs)
+
     def precheck(self) -> None:
         try:
-            get_astra_collection(
-                connection_config=self.connection_config,
-                collection_name=self.upload_config.collection_name,
-                keyspace=self.upload_config.keyspace,
-            ).options()
+            if self.upload_config.collection_name:
+                self.get_collection(collection_name=self.upload_config.collection_name).options()
+            else:
+                # check for db connection only if collection name is not provided
+                get_astra_db(
+                    connection_config=self.connection_config,
+                    keyspace=self.upload_config.keyspace,
+                )
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["astrapy"], extras="astradb")
-    def get_collection(self) -> "AstraDBCollection":
+    def get_collection(self, collection_name: Optional[str] = None) -> "AstraDBCollection":
         return get_astra_collection(
             connection_config=self.connection_config,
-            collection_name=self.upload_config.collection_name,
+            collection_name=collection_name or self.upload_config.collection_name,
             keyspace=self.upload_config.keyspace,
         )
 
+    def _collection_exists(self, collection_name: str):
+        from astrapy.exceptions import CollectionNotFoundException
+
+        collection = get_astra_collection(
+            connection_config=self.connection_config,
+            collection_name=collection_name,
+            keyspace=self.upload_config.keyspace,
+        )
+
+        try:
+            collection.options()
+            return True
+        except CollectionNotFoundException:
+            return False
+        except Exception as e:
+            logger.error(f"failed to check if astra collection exists : {e}")
+            raise DestinationConnectionError(f"failed to check if astra collection exists : {e}")
+
+    def format_destination_name(self, destination_name: str) -> str:
+        # AstraDB collection naming requirements:
+        # must be below 50 characters
+        # must be lowercase alphanumeric and underscores only
+        formatted = re.sub(r"[^a-z0-9]", "_", destination_name.lower())
+        return formatted
+
+    def create_destination(
+        self,
+        vector_length: int,
+        destination_name: str = "unstructuredautocreated",
+        similarity_metric: Optional[str] = "cosine",
+        **kwargs: Any,
+    ) -> bool:
+        destination_name = self.format_destination_name(destination_name)
+        collection_name = self.upload_config.collection_name or destination_name
+        self.upload_config.collection_name = collection_name
+
+        if not self._collection_exists(collection_name):
+            astra_db = get_astra_db(
+                connection_config=self.connection_config, keyspace=self.upload_config.keyspace
+            )
+            logger.info(
+                f"creating default astra collection '{collection_name}' with dimension "
+                f"{vector_length} and metric {similarity_metric}"
+            )
+            astra_db.create_collection(
+                collection_name,
+                dimension=vector_length,
+                metric=similarity_metric,
+            )
+            return True
+        logger.debug(f"collection with name '{collection_name}' already exists, skipping creation")
+        return False
+
     def delete_by_record_id(self, collection: "AstraDBCollection", file_data: FileData):
         logger.debug(
             f"deleting records from collection {collection.name} "

unstructured_ingest/v2/processes/connectors/confluence.py

@@ -8,6 +8,7 @@ from pydantic import Field, Secret
 from unstructured_ingest.error import SourceConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.html import HtmlMixin
+from unstructured_ingest.utils.string_and_date_utils import fix_unescaped_unicode
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -224,7 +225,6 @@ class ConfluenceDownloader(Downloader):
                     page_id=doc_id,
                     expand="history.lastUpdated,version,body.view",
                 )
-
         except Exception as e:
             logger.error(f"Failed to retrieve page with ID {doc_id}: {e}", exc_info=True)
             raise SourceConnectionError(f"Failed to retrieve page with ID {doc_id}: {e}")
@@ -233,10 +233,10 @@ class ConfluenceDownloader(Downloader):
             raise ValueError(f"Page with ID {doc_id} does not exist.")
 
         content = page["body"]["view"]["value"]
-        # This supports v2 html parsing in unstructured
         title = page["title"]
-        title_html = f"<title>{title}</title>"
-        content = f"<body class='Document' >{title_html}{content}</body>"
+        # Using h1 for title is supported by both v1 and v2 html parsing in unstructured
+        title_html = f"<h1>{title}</h1>"
+        content = fix_unescaped_unicode(f"<body class='Document' >{title_html}{content}</body>")
         if self.download_config.extract_images:
             with self.connection_config.get_client() as client:
                 content = self.download_config.extract_html_images(

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -92,6 +92,7 @@ class DeltaTableUploadStager(UploadStager):
         output_path = Path(output_dir) / Path(f"{output_filename}.parquet")
 
         df = convert_to_pandas_dataframe(elements_dict=elements_contents)
+        df = df.dropna(axis=1, how="all")
         df.to_parquet(output_path)
 
         return output_path
@@ -153,6 +154,7 @@ class DeltaTableUploader(Uploader):
             "table_or_uri": updated_upload_path,
             "data": df,
             "mode": "overwrite",
+            "schema_mode": "merge",
             "storage_options": storage_options,
         }
         queue = Queue()