unstructured-ingest 0.2.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

test/integration/connectors/sql/test_singlestore.py

@@ -0,0 +1,156 @@
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+
+import pandas as pd
+import pytest
+import singlestoredb as s2
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation import (
+    ValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
+    CONNECTOR_TYPE,
+    SingleStoreAccessConfig,
+    SingleStoreConnectionConfig,
+    SingleStoreDownloader,
+    SingleStoreDownloaderConfig,
+    SingleStoreIndexer,
+    SingleStoreIndexerConfig,
+    SingleStoreUploader,
+    SingleStoreUploaderConfig,
+    SingleStoreUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+@contextmanager
+def singlestore_download_setup(connect_params: dict) -> None:
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "source"
+    ):
+        with s2.connect(**connect_params) as connection:
+            with connection.cursor() as cursor:
+                for i in range(SEED_DATA_ROWS):
+                    sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+                    cursor.execute(sql_statment)
+                connection.commit()
+        yield
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_singlestore_source():
+    connect_params = {
+        "host": "localhost",
+        "port": 3306,
+        "database": "ingest_test",
+        "user": "root",
+        "password": "password",
+    }
+    with singlestore_download_setup(connect_params=connect_params):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            connection_config = SingleStoreConnectionConfig(
+                host=connect_params["host"],
+                port=connect_params["port"],
+                database=connect_params["database"],
+                user=connect_params["user"],
+                access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+            )
+            indexer = SingleStoreIndexer(
+                connection_config=connection_config,
+                index_config=SingleStoreIndexerConfig(
+                    table_name="cars", id_column="car_id", batch_size=5
+                ),
+            )
+            downloader = SingleStoreDownloader(
+                connection_config=connection_config,
+                download_config=SingleStoreDownloaderConfig(
+                    fields=["car_id", "brand"], download_dir=Path(tmpdir)
+                ),
+            )
+            await source_connector_validation(
+                indexer=indexer,
+                downloader=downloader,
+                configs=ValidationConfigs(
+                    test_id="singlestore",
+                    expected_num_files=SEED_DATA_ROWS,
+                    expected_number_indexed_file_data=4,
+                    validate_downloaded_files=True,
+                ),
+            )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+):
+    with s2.connect(**connect_params) as connection:
+        with connection.cursor() as cursor:
+            query = "select count(*) from elements;"
+            cursor.execute(query)
+            count = cursor.fetchone()[0]
+            assert (
+                count == expected_num_elements
+            ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+async def test_singlestore_destination(upload_file: Path):
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "singlestore" / "destination"
+    ):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            stager = SingleStoreUploadStager()
+            stager_params = {
+                "elements_filepath": upload_file,
+                "file_data": mock_file_data,
+                "output_dir": Path(tmpdir),
+                "output_filename": "test_db",
+            }
+            if stager.is_async():
+                staged_path = await stager.run_async(**stager_params)
+            else:
+                staged_path = stager.run(**stager_params)
+
+            # The stager should append the `.json` suffix to the output filename passed in.
+            assert staged_path.name == "test_db.json"
+
+            connect_params = {
+                "host": "localhost",
+                "port": 3306,
+                "database": "ingest_test",
+                "user": "root",
+                "password": "password",
+            }
+
+            uploader = SingleStoreUploader(
+                connection_config=SingleStoreConnectionConfig(
+                    host=connect_params["host"],
+                    port=connect_params["port"],
+                    database=connect_params["database"],
+                    user=connect_params["user"],
+                    access_config=SingleStoreAccessConfig(password=connect_params["password"]),
+                ),
+                upload_config=SingleStoreUploaderConfig(
+                    table_name="elements",
+                ),
+            )
+            if uploader.is_async():
+                await uploader.run_async(path=staged_path, file_data=mock_file_data)
+            else:
+                uploader.run(path=staged_path, file_data=mock_file_data)
+
+            staged_df = pd.read_json(staged_path, orient="records", lines=True)
+            expected_num_elements = len(staged_df)
+            validate_destination(
+                connect_params=connect_params,
+                expected_num_elements=expected_num_elements,
+            )

test/integration/connectors/test_s3.py

@@ -85,7 +85,7 @@ async def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):
 async def test_s3_minio_source(anon_connection_config: S3ConnectionConfig):
     anon_connection_config.endpoint_url = "http://localhost:9000"
     indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/")
-    with docker_compose_context(docker_compose_path=env_setup_path / "minio"):
+    with docker_compose_context(docker_compose_path=env_setup_path / "minio" / "source"):
         with tempfile.TemporaryDirectory() as tempdir:
             tempdir_path = Path(tempdir)
             download_config = S3DownloaderConfig(download_dir=tempdir_path)

test/integration/connectors/utils/docker_compose.py

@@ -3,6 +3,23 @@ from contextlib import contextmanager
 from pathlib import Path
 
 
+def docker_compose_down(docker_compose_path: Path):
+    cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v --rmi all"
+    print(f"Running command: {cmd}")
+    final_resp = subprocess.run(
+        cmd,
+        shell=True,
+        capture_output=True,
+    )
+    if final_resp.returncode != 0:
+        print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
+        print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
+
+
+def run_cleanup(docker_compose_path: Path):
+    docker_compose_down(docker_compose_path=docker_compose_path)
+
+
 @contextmanager
 def docker_compose_context(docker_compose_path: Path):
     # Dynamically run a specific docker compose file and make sure it gets cleanup by
@@ -30,15 +47,13 @@ def docker_compose_context(docker_compose_path: Path):
         if resp:
             print("STDOUT: {}".format(resp.stdout.decode("utf-8")))
             print("STDERR: {}".format(resp.stderr.decode("utf-8")))
-        raise e
-    finally:
-        cmd = f"docker compose -f {docker_compose_path.resolve()} down --remove-orphans -v"
-        print(f"Running command: {cmd}")
-        final_resp = subprocess.run(
+        cmd = f"docker compose -f {docker_compose_path.resolve()} logs"
+        logs = subprocess.run(
             cmd,
             shell=True,
             capture_output=True,
         )
-        if final_resp.returncode != 0:
-            print("STDOUT: {}".format(final_resp.stdout.decode("utf-8")))
-            print("STDERR: {}".format(final_resp.stderr.decode("utf-8")))
+        print("DOCKER LOGS: {}".format(logs.stdout.decode("utf-8")))
+        raise e
+    finally:
+        run_cleanup(docker_compose_path=docker_compose_path)

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.2.0"  # pragma: no cover
+__version__ = "0.2.1"  # pragma: no cover

unstructured_ingest/v2/interfaces/file_data.py

@@ -43,6 +43,7 @@ class FileData(DataClassJsonMixin):
     additional_metadata: dict[str, Any] = field(default_factory=dict)
     reprocess: bool = False
     local_download_path: Optional[str] = None
+    display_name: Optional[str] = None
 
     @classmethod
     def from_file(cls, path: str) -> "FileData":

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -11,7 +11,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
 from .airtable import CONNECTOR_TYPE as AIRTABLE_CONNECTOR_TYPE
 from .airtable import airtable_source_entry
 from .astradb import CONNECTOR_TYPE as ASTRA_DB_CONNECTOR_TYPE
-from .astradb import astra_db_destination_entry
+from .astradb import astra_db_destination_entry, astra_db_source_entry
 from .azure_cognitive_search import CONNECTOR_TYPE as AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE
 from .azure_cognitive_search import azure_cognitive_search_destination_entry
 from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
@@ -44,13 +44,12 @@ from .salesforce import CONNECTOR_TYPE as SALESFORCE_CONNECTOR_TYPE
 from .salesforce import salesforce_source_entry
 from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
 from .sharepoint import sharepoint_source_entry
-from .singlestore import CONNECTOR_TYPE as SINGLESTORE_CONNECTOR_TYPE
-from .singlestore import singlestore_destination_entry
 from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
 from .slack import slack_source_entry
 from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
 from .weaviate import weaviate_destination_entry
 
+add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
 add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
 
 add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_destination_entry)
@@ -88,9 +87,7 @@ add_source_entry(source_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_source_entry)
 
 add_destination_entry(destination_type=PINECONE_CONNECTOR_TYPE, entry=pinecone_destination_entry)
 add_source_entry(source_type=SHAREPOINT_CONNECTOR_TYPE, entry=sharepoint_source_entry)
-add_destination_entry(
-    destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
-)
+
 add_destination_entry(destination_type=MILVUS_CONNECTOR_TYPE, entry=milvus_destination_entry)
 add_destination_entry(
     destination_type=AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE,

unstructured_ingest/v2/processes/connectors/astradb.py

@@ -1,31 +1,50 @@
+import copy
+import csv
+import hashlib
 import json
+import sys
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from pydantic import Field, Secret
 
 from unstructured_ingest import __name__ as integration_name
 from unstructured_ingest.__version__ import __version__ as integration_version
-from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+)
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
+    Indexer,
+    IndexerConfig,
     Uploader,
     UploaderConfig,
     UploadStager,
     UploadStagerConfig,
+    download_responses,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
+    SourceRegistryEntry,
 )
 
 if TYPE_CHECKING:
+    from astrapy import AsyncCollection as AstraDBAsyncCollection
     from astrapy import Collection as AstraDBCollection
+    from astrapy import DataAPIClient as AstraDBClient
 
 
 CONNECTOR_TYPE = "astradb"
@@ -37,14 +56,253 @@ class AstraDBAccessConfig(AccessConfig):
 
 
 class AstraDBConnectionConfig(ConnectionConfig):
-    connection_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
     access_config: Secret[AstraDBAccessConfig]
 
+    @requires_dependencies(["astrapy"], extras="astradb")
+    def get_client(self) -> "AstraDBClient":
+        from astrapy import DataAPIClient as AstraDBClient
+
+        # Create a client object to interact with the Astra DB
+        # caller_name/version for Astra DB tracking
+        return AstraDBClient(
+            caller_name=integration_name,
+            caller_version=integration_version,
+        )
+
+
+def get_astra_collection(
+    connection_config: AstraDBConnectionConfig,
+    collection_name: str,
+    keyspace: str,
+) -> "AstraDBCollection":
+    # Build the Astra DB object.
+    access_configs = connection_config.access_config.get_secret_value()
+
+    # Create a client object to interact with the Astra DB
+    # caller_name/version for Astra DB tracking
+    client = connection_config.get_client()
+
+    # Get the database object
+    astra_db = client.get_database(
+        api_endpoint=access_configs.api_endpoint,
+        token=access_configs.token,
+        keyspace=keyspace,
+    )
+
+    # Connect to the collection
+    astra_db_collection = astra_db.get_collection(name=collection_name)
+    return astra_db_collection
+
+
+async def get_async_astra_collection(
+    connection_config: AstraDBConnectionConfig,
+    collection_name: str,
+    keyspace: str,
+) -> "AstraDBAsyncCollection":
+    # Build the Astra DB object.
+    access_configs = connection_config.access_config.get_secret_value()
+
+    # Create a client object to interact with the Astra DB
+    client = connection_config.get_client()
+
+    # Get the async database object
+    async_astra_db = client.get_async_database(
+        api_endpoint=access_configs.api_endpoint,
+        token=access_configs.token,
+        keyspace=keyspace,
+    )
+
+    # Get async collection from AsyncDatabase
+    async_astra_db_collection = await async_astra_db.get_collection(name=collection_name)
+    return async_astra_db_collection
+
 
 class AstraDBUploadStagerConfig(UploadStagerConfig):
     pass
 
 
+class AstraDBIndexerConfig(IndexerConfig):
+    collection_name: str = Field(
+        description="The name of the Astra DB collection. "
+        "Note that the collection name must only include letters, "
+        "numbers, and underscores."
+    )
+    keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
+    namespace: Optional[str] = Field(
+        default=None,
+        description="The Astra DB connection namespace.",
+        deprecated="Please use 'keyspace' instead.",
+    )
+    batch_size: int = Field(default=20, description="Number of records per batch")
+
+
+class AstraDBDownloaderConfig(DownloaderConfig):
+    fields: list[str] = field(default_factory=list)
+
+
+class AstraDBUploaderConfig(UploaderConfig):
+    collection_name: str = Field(
+        description="The name of the Astra DB collection. "
+        "Note that the collection name must only include letters, "
+        "numbers, and underscores."
+    )
+    embedding_dimension: int = Field(
+        default=384, description="The dimensionality of the embeddings"
+    )
+    keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
+    namespace: Optional[str] = Field(
+        default=None,
+        description="The Astra DB connection namespace.",
+        deprecated="Please use 'keyspace' instead.",
+    )
+    requested_indexing_policy: Optional[dict[str, Any]] = Field(
+        default=None,
+        description="The indexing policy to use for the collection.",
+        examples=['{"deny": ["metadata"]}'],
+    )
+    batch_size: int = Field(default=20, description="Number of records per batch")
+
+
+@dataclass
+class AstraDBIndexer(Indexer):
+    connection_config: AstraDBConnectionConfig
+    index_config: AstraDBIndexerConfig
+
+    def get_collection(self) -> "AstraDBCollection":
+        return get_astra_collection(
+            connection_config=self.connection_config,
+            collection_name=self.index_config.collection_name,
+            keyspace=self.index_config.keyspace or self.index_config.namespace,
+        )
+
+    def precheck(self) -> None:
+        try:
+            self.get_collection()
+        except Exception as e:
+            logger.error(f"Failed to validate connection {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+    def _get_doc_ids(self) -> set[str]:
+        """Fetches all document ids in an index"""
+        # Initialize set of ids
+        ids = set()
+
+        # Get the collection
+        collection = self.get_collection()
+
+        # Perform the find operation to get all items
+        astra_db_docs_cursor = collection.find({}, projection={"_id": True})
+
+        # Iterate over the cursor
+        astra_db_docs = []
+        for result in astra_db_docs_cursor:
+            astra_db_docs.append(result)
+
+        # Create file data for each astra record
+        for astra_record in astra_db_docs:
+            ids.add(astra_record["_id"])
+
+        return ids
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        all_ids = self._get_doc_ids()
+        ids = list(all_ids)
+        id_batches = batch_generator(ids, self.index_config.batch_size)
+
+        for batch in id_batches:
+            # Make sure the hash is always a positive number to create identified
+            identified = str(hash(batch) + sys.maxsize + 1)
+            fd = FileData(
+                identifier=identified,
+                connector_type=CONNECTOR_TYPE,
+                doc_type="batch",
+                metadata=FileDataSourceMetadata(
+                    date_processed=str(time()),
+                ),
+                additional_metadata={
+                    "ids": list(batch),
+                    "collection_name": self.index_config.collection_name,
+                    "keyspace": self.index_config.keyspace or self.index_config.namespace,
+                },
+            )
+            yield fd
+
+
+@dataclass
+class AstraDBDownloader(Downloader):
+    connection_config: AstraDBConnectionConfig
+    download_config: AstraDBDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def is_async(self) -> bool:
+        return True
+
+    def get_identifier(self, record_id: str) -> str:
+        f = f"{record_id}"
+        if self.download_config.fields:
+            f = "{}-{}".format(
+                f,
+                hashlib.sha256(",".join(self.download_config.fields).encode()).hexdigest()[:8],
+            )
+        return f
+
+    def write_astra_result_to_csv(self, astra_result: dict, download_path: str) -> None:
+        with open(download_path, "w", encoding="utf8") as f:
+            writer = csv.writer(f)
+            writer.writerow(astra_result.keys())
+            writer.writerow(astra_result.values())
+
+    def generate_download_response(self, result: dict, file_data: FileData) -> DownloadResponse:
+        record_id = result["_id"]
+        filename_id = self.get_identifier(record_id=record_id)
+        filename = f"{filename_id}.csv"  # csv to preserve column info
+        download_path = self.download_dir / Path(filename)
+        logger.debug(f"Downloading results from record {record_id} as csv to {download_path}")
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            self.write_astra_result_to_csv(astra_result=result, download_path=download_path)
+        except Exception as e:
+            logger.error(
+                f"failed to download from record {record_id} to {download_path}: {e}",
+                exc_info=True,
+            )
+            raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
+
+        # modify input file_data for download_response
+        copied_file_data = copy.deepcopy(file_data)
+        copied_file_data.identifier = filename
+        copied_file_data.doc_type = "file"
+        copied_file_data.metadata.date_processed = str(time())
+        copied_file_data.metadata.record_locator = {"document_id": record_id}
+        copied_file_data.additional_metadata.pop("ids", None)
+        return super().generate_download_response(
+            file_data=copied_file_data, download_path=download_path
+        )
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        raise NotImplementedError("Use astradb run_async instead")
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        # Get metadata from file_data
+        ids: list[str] = file_data.additional_metadata["ids"]
+        collection_name: str = file_data.additional_metadata["collection_name"]
+        keyspace: str = file_data.additional_metadata["keyspace"]
+
+        # Retrieve results from async collection
+        download_responses = []
+        async_astra_collection = await get_async_astra_collection(
+            connection_config=self.connection_config,
+            collection_name=collection_name,
+            keyspace=keyspace,
+        )
+        async for result in async_astra_collection.find({"_id": {"$in": ids}}):
+            download_responses.append(
+                self.generate_download_response(result=result, file_data=file_data)
+            )
+        return download_responses
+
+
 @dataclass
 class AstraDBUploadStager(UploadStager):
     upload_stager_config: AstraDBUploadStagerConfig = field(
@@ -77,29 +335,6 @@ class AstraDBUploadStager(UploadStager):
         return output_path
 
 
-class AstraDBUploaderConfig(UploaderConfig):
-    collection_name: str = Field(
-        description="The name of the Astra DB collection. "
-        "Note that the collection name must only include letters, "
-        "numbers, and underscores."
-    )
-    embedding_dimension: int = Field(
-        default=384, description="The dimensionality of the embeddings"
-    )
-    keyspace: Optional[str] = Field(default=None, description="The Astra DB connection keyspace.")
-    namespace: Optional[str] = Field(
-        default=None,
-        description="The Astra DB connection namespace.",
-        deprecated="Please use 'keyspace' instead.",
-    )
-    requested_indexing_policy: Optional[dict[str, Any]] = Field(
-        default=None,
-        description="The indexing policy to use for the collection.",
-        examples=['{"deny": ["metadata"]}'],
-    )
-    batch_size: int = Field(default=20, description="Number of records per batch")
-
-
 @dataclass
 class AstraDBUploader(Uploader):
     connection_config: AstraDBConnectionConfig
@@ -108,43 +343,23 @@ class AstraDBUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            self.get_collection()
+            get_astra_collection(
+                connection_config=self.connection_config,
+                collection_name=self.upload_config.collection_name,
+                keyspace=self.upload_config.keyspace or self.upload_config.namespace,
+            )
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["astrapy"], extras="astradb")
     def get_collection(self) -> "AstraDBCollection":
-        from astrapy import DataAPIClient as AstraDBClient
-
-        # Choose keyspace or deprecated namespace
-        keyspace_param = self.upload_config.keyspace or self.upload_config.namespace
-
-        # Get the collection_name
-        collection_name = self.upload_config.collection_name
-
-        # Build the Astra DB object.
-        access_configs = self.connection_config.access_config.get_secret_value()
-
-        # Create a client object to interact with the Astra DB
-        # caller_name/version for Astra DB tracking
-        my_client = AstraDBClient(
-            caller_name=integration_name,
-            caller_version=integration_version,
-        )
-
-        # Get the database object
-        astra_db = my_client.get_database(
-            api_endpoint=access_configs.api_endpoint,
-            token=access_configs.token,
-            keyspace=keyspace_param,
+        return get_astra_collection(
+            connection_config=self.connection_config,
+            collection_name=self.upload_config.collection_name,
+            keyspace=self.upload_config.keyspace or self.upload_config.namespace,
         )
 
-        # Connect to the newly created collection
-        astra_db_collection = astra_db.get_collection(name=collection_name)
-
-        return astra_db_collection
-
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         with path.open("r") as file:
             elements_dict = json.load(file)
@@ -160,6 +375,14 @@ class AstraDBUploader(Uploader):
             collection.insert_many(chunk)
 
 
+astra_db_source_entry = SourceRegistryEntry(
+    indexer=AstraDBIndexer,
+    indexer_config=AstraDBIndexerConfig,
+    downloader=AstraDBDownloader,
+    downloader_config=AstraDBDownloaderConfig,
+    connection_config=AstraDBConnectionConfig,
+)
+
 astra_db_destination_entry = DestinationRegistryEntry(
     connection_config=AstraDBConnectionConfig,
     upload_stager_config=AstraDBUploadStagerConfig,

unstructured_ingest/v2/processes/connectors/databricks/volumes.py

@@ -166,7 +166,9 @@ class DatabricksVolumesUploader(Uploader, ABC):
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        output_path = os.path.join(self.upload_config.path, file_data.source_identifiers.filename)
+        output_path = os.path.join(
+            self.upload_config.path, f"{file_data.source_identifiers.filename}.json"
+        )
         with open(path, "rb") as elements_file:
             self.connection_config.get_client().files.upload(
                 file_path=output_path,

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -176,6 +176,7 @@ class FsspecIndexer(Indexer):
                 ),
                 metadata=self.get_metadata(file_data=file_data),
                 additional_metadata=additional_metadata,
+                display_name=file_path,
             )