unstructured-ingest 0.5.0__tar.gz → 0.5.1__tar.gz

{unstructured_ingest-0.5.0/unstructured_ingest.egg-info → unstructured_ingest-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.5.0
+Version: 0.5.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: python-dateutil
+Requires-Dist: tqdm
 Requires-Dist: pydantic>=2.7
 Requires-Dist: pandas
-Requires-Dist: click
 Requires-Dist: dataclasses_json
-Requires-Dist: tqdm
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: python-dateutil
+Requires-Dist: click
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -66,16 +66,16 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -86,31 +86,31 @@ Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,23 +124,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: backoff; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: bs4; extra == "onedrive"
-Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -152,21 +152,21 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -182,9 +182,9 @@ Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
+Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"
 Requires-Dist: vastdb; extra == "vastdb"
-Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai

unstructured_ingest-0.5.1/test/integration/connectors/sql/test_vastdb.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+import pytest
+from _pytest.fixtures import TopRequest
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
+from unstructured_ingest.v2.processes.connectors.sql.vastdb import (
+    CONNECTOR_TYPE,
+    VastdbUploadStager,
+    VastdbUploadStagerConfig,
+)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_vast_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = VastdbUploadStager(
+        upload_stager_config=VastdbUploadStagerConfig(rename_columns_map={"page_number": "page"})
+    )
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

unstructured_ingest-0.5.1/test/integration/connectors/test_google_drive.py

@@ -0,0 +1,116 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+    UNCATEGORIZED_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    get_all_file_data,
+    run_all_validations,
+    update_fixtures,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import Downloader, Indexer
+from unstructured_ingest.v2.processes.connectors.google_drive import (
+    CONNECTOR_TYPE,
+    GoogleDriveAccessConfig,
+    GoogleDriveConnectionConfig,
+    GoogleDriveDownloader,
+    GoogleDriveDownloaderConfig,
+    GoogleDriveIndexer,
+    GoogleDriveIndexerConfig,
+)
+
+
+@requires_env("GOOGLE_DRIVE_SERVICE_KEY")
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE)
+def test_google_drive_source(temp_dir):
+    # Retrieve environment variables
+    service_account_key = os.environ["GOOGLE_DRIVE_SERVICE_KEY"]
+
+    # Create connection and indexer configurations
+    access_config = GoogleDriveAccessConfig(service_account_key=service_account_key)
+    connection_config = GoogleDriveConnectionConfig(
+        drive_id="1XidSOO76VpZ4m0i3gJN2m1X0Obol3UAi",
+        access_config=access_config,
+    )
+    index_config = GoogleDriveIndexerConfig(recursive=True)
+
+    download_config = GoogleDriveDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = GoogleDriveIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = GoogleDriveDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="google_drive_source",
+            expected_num_files=1,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+def source_connector_validation(
+    indexer: Indexer,
+    downloader: Downloader,
+    configs: SourceValidationConfigs,
+    overwrite_fixtures: bool = os.getenv("OVERWRITE_FIXTURES", "False").lower() == "true",
+) -> None:
+    # Run common validations on the process of running a source connector, supporting dynamic
+    # validators that get passed in along with comparisons on the saved expected values.
+    # If overwrite_fixtures is st to True, will ignore all validators but instead overwrite the
+    # expected values with what gets generated by this test.
+    all_predownload_file_data = []
+    all_postdownload_file_data = []
+    indexer.precheck()
+    download_dir = downloader.download_config.download_dir
+    test_output_dir = configs.test_output_dir()
+
+    for file_data in indexer.run():
+        assert file_data
+        predownload_file_data = file_data.model_copy(deep=True)
+        all_predownload_file_data.append(predownload_file_data)
+        resp = downloader.run(file_data=file_data)
+        if isinstance(resp, list):
+            for r in resp:
+                postdownload_file_data = r["file_data"].model_copy(deep=True)
+                all_postdownload_file_data.append(postdownload_file_data)
+        else:
+            postdownload_file_data = resp["file_data"].model_copy(deep=True)
+            all_postdownload_file_data.append(postdownload_file_data)
+
+    if not overwrite_fixtures:
+        print("Running validation")
+        run_all_validations(
+            configs=configs,
+            predownload_file_data=all_predownload_file_data,
+            postdownload_file_data=all_postdownload_file_data,
+            download_dir=download_dir,
+            test_output_dir=test_output_dir,
+        )
+    else:
+        print("Running fixtures update")
+        update_fixtures(
+            output_dir=test_output_dir,
+            download_dir=download_dir,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=all_predownload_file_data,
+                all_postdownload_file_data=all_postdownload_file_data,
+            ),
+            save_downloads=configs.validate_downloaded_files,
+            save_filedata=configs.validate_file_data,
+        )

unstructured_ingest-0.5.1/test/unit/v2/connectors/motherduck/test_base.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+import pytest
+from pytest_mock import MockerFixture
+
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces.file_data import SourceIdentifiers
+from unstructured_ingest.v2.interfaces.upload_stager import UploadStagerConfig
+from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+
+
+@pytest.fixture
+def mock_instance() -> BaseDuckDBUploadStager:
+    return BaseDuckDBUploadStager(UploadStagerConfig())
+
+
+@pytest.mark.parametrize(
+    ("input_filepath", "output_filename", "expected"),
+    [
+        (
+            "/path/to/input_file.ndjson",
+            "output_file.ndjson",
+            "output_file.ndjson",
+        ),
+        ("input_file.txt", "output_file.json", "output_file.txt"),
+        ("/path/to/input_file.json", "output_file", "output_file.json"),
+    ],
+)
+def test_run_output_filename_suffix(
+    mocker: MockerFixture,
+    mock_instance: BaseDuckDBUploadStager,
+    input_filepath: str,
+    output_filename: str,
+    expected: str,
+):
+    output_dir = Path("/tmp/test/output_dir")
+
+    # Mocks
+    mock_get_data = mocker.patch(
+        "unstructured_ingest.v2.processes.connectors.duckdb.base.get_data",
+        return_value=[{"key": "value"}, {"key": "value2"}],
+    )
+    mock_conform_dict = mocker.patch.object(
+        BaseDuckDBUploadStager,
+        "conform_dict",
+        side_effect=lambda element_dict, file_data: element_dict,
+    )
+    mock_get_output_path = mocker.patch.object(
+        BaseDuckDBUploadStager, "get_output_path", return_value=output_dir / expected
+    )
+    mock_write_output = mocker.patch(
+        "unstructured_ingest.v2.processes.connectors.duckdb.base.write_data", return_value=None
+    )
+
+    # Act
+    result = mock_instance.run(
+        elements_filepath=Path(input_filepath),
+        file_data=FileData(
+            identifier="test",
+            connector_type="test",
+            source_identifiers=SourceIdentifiers(filename=input_filepath, fullpath=input_filepath),
+        ),
+        output_dir=output_dir,
+        output_filename=output_filename,
+    )
+
+    # Assert
+    mock_get_data.assert_called_once_with(path=Path(input_filepath))
+    assert mock_conform_dict.call_count == 2
+    mock_get_output_path.assert_called_once_with(output_filename=expected, output_dir=output_dir)
+    mock_write_output.assert_called_once_with(
+        path=output_dir / expected, data=[{"key": "value"}, {"key": "value2"}]
+    )
+    assert result.name == expected

unstructured_ingest-0.5.1/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"  # pragma: no cover

{unstructured_ingest-0.5.0 → unstructured_ingest-0.5.1}/unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -81,6 +81,8 @@ class BaseDuckDBUploadStager(UploadStager):
         **kwargs: Any,
     ) -> Path:
         elements_contents = get_data(path=elements_filepath)
+        output_filename_suffix = Path(elements_filepath).suffix
+        output_filename = f"{Path(output_filename).stem}{output_filename_suffix}"
         output_path = self.get_output_path(output_filename=output_filename, output_dir=output_dir)
 
         output = [

{unstructured_ingest-0.5.0 → unstructured_ingest-0.5.1}/unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py

@@ -61,7 +61,7 @@ class MotherDuckConnectionConfig(ConnectionConfig):
                 "custom_user_agent": f"unstructured-io-ingest/{unstructured_io_ingest_version}"
             },
         ) as conn:
-            conn.sql(f"USE {self.database}")
+            conn.sql(f'USE "{self.database}"')
             yield conn
 
     @contextmanager
@@ -102,11 +102,12 @@ class MotherDuckUploader(Uploader):
 
     def upload_dataframe(self, df: pd.DataFrame) -> None:
         logger.debug(f"uploading {len(df)} entries to {self.connection_config.database} ")
+        database = self.connection_config.database
+        db_schema = self.connection_config.db_schema
+        table = self.connection_config.table
 
         with self.connection_config.get_client() as conn:
-            conn.query(
-                f"INSERT INTO {self.connection_config.db_schema}.{self.connection_config.table} BY NAME SELECT * FROM df"  # noqa: E501
-            )
+            conn.query(f'INSERT INTO "{database}"."{db_schema}"."{table}" BY NAME SELECT * FROM df')
 
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
         df = pd.DataFrame(data=data)

{unstructured_ingest-0.5.0 → unstructured_ingest-0.5.1}/unstructured_ingest/v2/processes/connectors/google_drive.py

@@ -310,20 +310,22 @@ class GoogleDriveDownloader(Downloader):
         from googleapiclient.http import MediaIoBaseDownload
 
         logger.debug(f"fetching file: {file_data.source_identifiers.fullpath}")
-        mime_type = file_data.additional_metadata["mimeType"]
         record_id = file_data.identifier
+        mime_type = file_data.additional_metadata["mimeType"]
+        if not mime_type:
+            raise TypeError(
+                f"File not supported. Name: {file_data.source_identifiers.filename} "
+                f"ID: {record_id} "
+                f"MimeType: {mime_type}"
+            )
         with self.connection_config.get_client() as client:
-            if mime_type.startswith("application/vnd.google-apps"):
+            if (
+                mime_type.startswith("application/vnd.google-apps")
+                and mime_type in GOOGLE_DRIVE_EXPORT_TYPES
+            ):
                 export_mime = GOOGLE_DRIVE_EXPORT_TYPES.get(
-                    self.meta.get("mimeType"),  # type: ignore
+                    mime_type,  # type: ignore
                 )
-                if not export_mime:
-                    raise TypeError(
-                        f"File not supported. Name: {file_data.source_identifiers.filename} "
-                        f"ID: {record_id} "
-                        f"MimeType: {mime_type}"
-                    )
-
                 request = client.export_media(
                     fileId=record_id,
                     mimeType=export_mime,

{unstructured_ingest-0.5.0 → unstructured_ingest-0.5.1}/unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -81,6 +81,7 @@ ALLOWED_FIELDS = (
     "link_urls",
     "link_texts",
     "text_as_html",
+    "entities",
 )
 
 

{unstructured_ingest-0.5.0 → unstructured_ingest-0.5.1}/unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -38,48 +38,6 @@ from unstructured_ingest.v2.interfaces import (
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.utils import get_enhanced_element_id
 
-_COLUMNS = (
-    "id",
-    "element_id",
-    "text",
-    "embeddings",
-    "type",
-    "system",
-    "layout_width",
-    "layout_height",
-    "points",
-    "url",
-    "version",
-    "date_created",
-    "date_modified",
-    "date_processed",
-    "permissions_data",
-    "record_locator",
-    "category_depth",
-    "parent_id",
-    "attached_filename",
-    "filetype",
-    "last_modified",
-    "file_directory",
-    "filename",
-    "languages",
-    "page_number",
-    "links",
-    "page_name",
-    "link_urls",
-    "link_texts",
-    "sent_from",
-    "sent_to",
-    "subject",
-    "section",
-    "header_footer_type",
-    "emphasized_text_contents",
-    "emphasized_text_tags",
-    "text_as_html",
-    "regex_metadata",
-    "detection_class_prob",
-)
-
 _DATE_COLUMNS = ("date_created", "date_modified", "date_processed", "last_modified")
 
 
@@ -270,10 +228,8 @@ class SQLUploadStager(UploadStager):
 
         data["id"] = get_enhanced_element_id(element_dict=data, file_data=file_data)
 
-        # remove extraneous, not supported columns
-        element = {k: v for k, v in data.items() if k in _COLUMNS}
-        element[RECORD_ID_LABEL] = file_data.identifier
-        return element
+        data[RECORD_ID_LABEL] = file_data.identifier
+        return data
 
     def conform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
         for column in filter(lambda x: x in df.columns, _DATE_COLUMNS):
@@ -375,7 +331,7 @@ class SQLUploader(Uploader):
         missing_columns = schema_fields - columns
 
         if columns_to_drop:
-            logger.warning(
+            logger.info(
                 "Following columns will be dropped to match the table's schema: "
                 f"{', '.join(columns_to_drop)}"
             )

{unstructured_ingest-0.5.0 → unstructured_ingest-0.5.1}/unstructured_ingest/v2/processes/connectors/sql/vastdb.py

@@ -19,7 +19,6 @@ from unstructured_ingest.v2.processes.connector_registry import (
     SourceRegistryEntry,
 )
 from unstructured_ingest.v2.processes.connectors.sql.sql import (
-    _COLUMNS,
     SQLAccessConfig,
     SqlBatchFileData,
     SQLConnectionConfig,
@@ -149,13 +148,11 @@ class VastdbUploadStagerConfig(SQLUploadStagerConfig):
         default=None,
         description="Map of column names to rename, ex: {'old_name': 'new_name'}",
     )
-    additional_columns: Optional[list[str]] = Field(
-        default_factory=list, description="Additional columns to include in the upload"
-    )
 
 
+@dataclass
 class VastdbUploadStager(SQLUploadStager):
-    upload_stager_config: VastdbUploadStagerConfig
+    upload_stager_config: VastdbUploadStagerConfig = field(default_factory=VastdbUploadStagerConfig)
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
@@ -168,13 +165,8 @@ class VastdbUploadStager(SQLUploadStager):
         data.update(coordinates)
 
         data["id"] = get_enhanced_element_id(element_dict=data, file_data=file_data)
-
-        # remove extraneous, not supported columns
-        # but also allow for additional columns
-        approved_columns = set(_COLUMNS).union(self.upload_stager_config.additional_columns)
-        element = {k: v for k, v in data.items() if k in approved_columns}
-        element[RECORD_ID_LABEL] = file_data.identifier
-        return element
+        data[RECORD_ID_LABEL] = file_data.identifier
+        return data
 
     def conform_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
         df = super().conform_dataframe(df=df)