unstructured-ingest 0.5.8__tar.gz → 0.5.9__tar.gz

{unstructured_ingest-0.5.8/unstructured_ingest.egg-info → unstructured_ingest-0.5.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.8
+Version: 0.5.9
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: python-dateutil
-Requires-Dist: pydantic>=2.7
-Requires-Dist: dataclasses_json
-Requires-Dist: pandas
 Requires-Dist: tqdm
+Requires-Dist: pandas
 Requires-Dist: click
+Requires-Dist: dataclasses_json
+Requires-Dist: python-dateutil
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: pydantic>=2.7
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -66,8 +66,8 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
@@ -86,8 +86,8 @@ Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -124,23 +124,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: backoff; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -155,8 +155,8 @@ Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,8 +178,8 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
@@ -188,8 +188,8 @@ Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

{unstructured_ingest-0.5.8 → unstructured_ingest-0.5.9}/test/integration/connectors/test_astradb.py

@@ -56,12 +56,7 @@ def test_precheck_succeeds_indexer(connection_config: AstraDBConnectionConfig):
         connection_config=connection_config,
         index_config=AstraDBIndexerConfig(collection_name=EXISTENT_COLLECTION_NAME),
     )
-    uploader = AstraDBUploader(
-        connection_config=connection_config,
-        upload_config=AstraDBUploaderConfig(collection_name=EXISTENT_COLLECTION_NAME),
-    )
     indexer.precheck()
-    uploader.precheck()
 
 
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
@@ -73,6 +68,12 @@ def test_precheck_succeeds_uploader(connection_config: AstraDBConnectionConfig):
     )
     uploader.precheck()
 
+    uploader2 = AstraDBUploader(
+        connection_config=connection_config,
+        upload_config=AstraDBUploaderConfig(),
+    )
+    uploader2.precheck()
+
 
 @pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, VECTOR_DB_TAG)
 @requires_env("ASTRA_DB_APPLICATION_TOKEN", "ASTRA_DB_API_ENDPOINT")
@@ -216,6 +217,32 @@ async def test_astra_search_destination(
     )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+@requires_env("ASTRA_DB_API_ENDPOINT", "ASTRA_DB_APPLICATION_TOKEN")
+def test_astra_create_destination():
+    env_data = get_env_data()
+    connection_config = AstraDBConnectionConfig(
+        access_config=AstraDBAccessConfig(api_endpoint=env_data.api_endpoint, token=env_data.token),
+    )
+    uploader = AstraDBUploader(
+        connection_config=connection_config,
+        upload_config=AstraDBUploaderConfig(),
+    )
+    collection_name = "system_created-123"
+    formatted_collection_name = "system_created_123"
+    created = uploader.create_destination(destination_name=collection_name, vector_length=3072)
+    assert created
+    assert uploader.upload_config.collection_name == formatted_collection_name
+
+    created = uploader.create_destination(destination_name=collection_name, vector_length=3072)
+    assert not created
+
+    # cleanup
+    client = AstraDBClient()
+    db = client.get_database(api_endpoint=env_data.api_endpoint, token=env_data.token)
+    db.drop_collection(formatted_collection_name)
+
+
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_astra_stager(

unstructured_ingest-0.5.9/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.9"  # pragma: no cover

{unstructured_ingest-0.5.8 → unstructured_ingest-0.5.9}/unstructured_ingest/v2/processes/connectors/astradb.py

@@ -1,5 +1,6 @@
 import csv
 import hashlib
+import re
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
@@ -48,6 +49,7 @@ if TYPE_CHECKING:
     from astrapy import AsyncCollection as AstraDBAsyncCollection
     from astrapy import Collection as AstraDBCollection
     from astrapy import DataAPIClient as AstraDBClient
+    from astrapy import Database as AstraDB
 
 
 CONNECTOR_TYPE = "astradb"
@@ -85,11 +87,10 @@ class AstraDBConnectionConfig(ConnectionConfig):
         )
 
 
-def get_astra_collection(
+def get_astra_db(
     connection_config: AstraDBConnectionConfig,
-    collection_name: str,
     keyspace: str,
-) -> "AstraDBCollection":
+) -> "AstraDB":
     # Build the Astra DB object.
     access_configs = connection_config.access_config.get_secret_value()
 
@@ -103,9 +104,20 @@ def get_astra_collection(
         token=access_configs.token,
         keyspace=keyspace,
     )
+    return astra_db
+
 
-    # Connect to the collection
+def get_astra_collection(
+    connection_config: AstraDBConnectionConfig,
+    collection_name: str,
+    keyspace: str,
+) -> "AstraDBCollection":
+
+    astra_db = get_astra_db(connection_config=connection_config, keyspace=keyspace)
+
+    # astradb will return a collection object in all cases (even if it doesn't exist)
     astra_db_collection = astra_db.get_collection(name=collection_name)
+
     return astra_db_collection
 
 
@@ -151,10 +163,11 @@ class AstraDBDownloaderConfig(DownloaderConfig):
 
 
 class AstraDBUploaderConfig(UploaderConfig):
-    collection_name: str = Field(
+    collection_name: Optional[str] = Field(
         description="The name of the Astra DB collection. "
         "Note that the collection name must only include letters, "
-        "numbers, and underscores."
+        "numbers, and underscores.",
+        default=None,
     )
     keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
     requested_indexing_policy: Optional[dict[str, Any]] = Field(
@@ -337,25 +350,84 @@ class AstraDBUploader(Uploader):
     upload_config: AstraDBUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    def init(self, **kwargs: Any) -> None:
+        self.create_destination(**kwargs)
+
     def precheck(self) -> None:
         try:
-            get_astra_collection(
-                connection_config=self.connection_config,
-                collection_name=self.upload_config.collection_name,
-                keyspace=self.upload_config.keyspace,
-            ).options()
+            if self.upload_config.collection_name:
+                self.get_collection(collection_name=self.upload_config.collection_name).options()
+            else:
+                # check for db connection only if collection name is not provided
+                get_astra_db(
+                    connection_config=self.connection_config,
+                    keyspace=self.upload_config.keyspace,
+                )
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["astrapy"], extras="astradb")
-    def get_collection(self) -> "AstraDBCollection":
+    def get_collection(self, collection_name: Optional[str] = None) -> "AstraDBCollection":
         return get_astra_collection(
             connection_config=self.connection_config,
-            collection_name=self.upload_config.collection_name,
+            collection_name=collection_name or self.upload_config.collection_name,
             keyspace=self.upload_config.keyspace,
         )
 
+    def _collection_exists(self, collection_name: str):
+        from astrapy.exceptions import CollectionNotFoundException
+
+        collection = get_astra_collection(
+            connection_config=self.connection_config,
+            collection_name=collection_name,
+            keyspace=self.upload_config.keyspace,
+        )
+
+        try:
+            collection.options()
+            return True
+        except CollectionNotFoundException:
+            return False
+        except Exception as e:
+            logger.error(f"failed to check if astra collection exists : {e}")
+            raise DestinationConnectionError(f"failed to check if astra collection exists : {e}")
+
+    def format_destination_name(self, destination_name: str) -> str:
+        # AstraDB collection naming requirements:
+        # must be below 50 characters
+        # must be lowercase alphanumeric and underscores only
+        formatted = re.sub(r"[^a-z0-9]", "_", destination_name.lower())
+        return formatted
+
+    def create_destination(
+        self,
+        vector_length: int,
+        destination_name: str = "unstructuredautocreated",
+        similarity_metric: Optional[str] = "cosine",
+        **kwargs: Any,
+    ) -> bool:
+        destination_name = self.format_destination_name(destination_name)
+        collection_name = self.upload_config.collection_name or destination_name
+        self.upload_config.collection_name = collection_name
+
+        if not self._collection_exists(collection_name):
+            astra_db = get_astra_db(
+                connection_config=self.connection_config, keyspace=self.upload_config.keyspace
+            )
+            logger.info(
+                f"creating default astra collection '{collection_name}' with dimension "
+                f"{vector_length} and metric {similarity_metric}"
+            )
+            astra_db.create_collection(
+                collection_name,
+                dimension=vector_length,
+                metric=similarity_metric,
+            )
+            return True
+        logger.debug(f"collection with name '{collection_name}' already exists, skipping creation")
+        return False
+
     def delete_by_record_id(self, collection: "AstraDBCollection", file_data: FileData):
         logger.debug(
             f"deleting records from collection {collection.name} "

{unstructured_ingest-0.5.8 → unstructured_ingest-0.5.9}/unstructured_ingest/v2/processes/connectors/confluence.py

@@ -233,9 +233,9 @@ class ConfluenceDownloader(Downloader):
             raise ValueError(f"Page with ID {doc_id} does not exist.")
 
         content = page["body"]["view"]["value"]
-        # This supports v2 html parsing in unstructured
         title = page["title"]
-        title_html = f"<title>{title}</title>"
+        # Using h1 for title is supported by both v1 and v2 html parsing in unstructured
+        title_html = f"<h1>{title}</h1>"
         content = f"<body class='Document' >{title_html}{content}</body>"
         if self.download_config.extract_images:
             with self.connection_config.get_client() as client:

{unstructured_ingest-0.5.8 → unstructured_ingest-0.5.9/unstructured_ingest.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: unstructured-ingest
-Version: 0.5.8
+Version: 0.5.9
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: python-dateutil
-Requires-Dist: pydantic>=2.7
-Requires-Dist: dataclasses_json
-Requires-Dist: pandas
 Requires-Dist: tqdm
+Requires-Dist: pandas
 Requires-Dist: click
+Requires-Dist: dataclasses_json
+Requires-Dist: python-dateutil
 Requires-Dist: opentelemetry-sdk
+Requires-Dist: pydantic>=2.7
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -66,8 +66,8 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
@@ -86,8 +86,8 @@ Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
@@ -124,23 +124,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
+Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
-Requires-Dist: neo4j-rust-ext; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: backoff; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: httpx; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
+Requires-Dist: backoff; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -155,8 +155,8 @@ Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
@@ -165,8 +165,8 @@ Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,8 +178,8 @@ Requires-Dist: databricks-sql-connector; extra == "databricks-delta-tables"
 Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
-Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
 Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
 Requires-Dist: ibis; extra == "vastdb"
@@ -188,8 +188,8 @@ Requires-Dist: pyarrow; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: boto3; extra == "bedrock"
 Requires-Dist: aioboto3; extra == "bedrock"
+Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 Dynamic: author

unstructured_ingest-0.5.8/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "0.5.8"  # pragma: no cover