unstructured-ingest 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -9,6 +9,7 @@ from pydantic import Field, Secret
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.data_prep import flatten_dict, generator_batching_wbytes
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -23,6 +24,7 @@ from unstructured_ingest.v2.processes.connector_registry import DestinationRegis
 
 if TYPE_CHECKING:
     from pinecone import Index as PineconeIndex
+    from pinecone import Pinecone
 
 
 CONNECTOR_TYPE = "pinecone"
@@ -43,16 +45,19 @@ class PineconeConnectionConfig(ConnectionConfig):
     )
 
     @requires_dependencies(["pinecone"], extras="pinecone")
-    def get_index(self, **index_kwargs) -> "PineconeIndex":
+    def get_client(self, **index_kwargs) -> "Pinecone":
         from pinecone import Pinecone
 
         from unstructured_ingest import __version__ as unstructured_version
 
-        pc = Pinecone(
+        return Pinecone(
             api_key=self.access_config.get_secret_value().pinecone_api_key,
             source_tag=f"unstructured_ingest=={unstructured_version}",
         )
 
+    def get_index(self, **index_kwargs) -> "PineconeIndex":
+        pc = self.get_client()
+
         index = pc.Index(name=self.index_name, **index_kwargs)
         logger.debug(f"connected to index: {pc.describe_index(self.index_name)}")
         return index
@@ -106,7 +111,7 @@ class PineconeUploadStager(UploadStager):
         default_factory=lambda: PineconeUploadStagerConfig()
     )
 
-    def conform_dict(self, element_dict: dict) -> dict:
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         embeddings = element_dict.pop("embeddings", None)
         metadata: dict[str, Any] = element_dict.pop("metadata", {})
         data_source = metadata.pop("data_source", {})
@@ -121,19 +126,23 @@ class PineconeUploadStager(UploadStager):
                 }
             )
 
+        metadata = flatten_dict(
+            pinecone_metadata,
+            separator="-",
+            flatten_lists=True,
+            remove_none=True,
+        )
+        metadata[RECORD_ID_LABEL] = file_data.identifier
+
         return {
             "id": str(uuid.uuid4()),
             "values": embeddings,
-            "metadata": flatten_dict(
-                pinecone_metadata,
-                separator="-",
-                flatten_lists=True,
-                remove_none=True,
-            ),
+            "metadata": metadata,
         }
 
     def run(
         self,
+        file_data: FileData,
         elements_filepath: Path,
         output_dir: Path,
         output_filename: str,
@@ -143,10 +152,15 @@ class PineconeUploadStager(UploadStager):
             elements_contents = json.load(elements_file)
 
         conformed_elements = [
-            self.conform_dict(element_dict=element) for element in elements_contents
+            self.conform_dict(element_dict=element, file_data=file_data)
+            for element in elements_contents
         ]
 
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        if Path(output_filename).suffix != ".json":
+            output_filename = f"{output_filename}.json"
+        else:
+            output_filename = f"{Path(output_filename).stem}.json"
+        output_path = Path(output_dir) / Path(f"{output_filename}")
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
         with open(output_path, "w") as output_file:
@@ -167,6 +181,55 @@ class PineconeUploader(Uploader):
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
+    def pod_delete_by_record_id(self, file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"from pinecone pod index"
+        )
+        index = self.connection_config.get_index(pool_threads=MAX_POOL_THREADS)
+        delete_kwargs = {"filter": {RECORD_ID_LABEL: {"$eq": file_data.identifier}}}
+        if namespace := self.upload_config.namespace:
+            delete_kwargs["namespace"] = namespace
+
+        resp = index.delete(**delete_kwargs)
+        logger.debug(
+            f"deleted any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"from pinecone index: {resp}"
+        )
+
+    def serverless_delete_by_record_id(self, file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"from pinecone serverless index"
+        )
+        index = self.connection_config.get_index(pool_threads=MAX_POOL_THREADS)
+        index_stats = index.describe_index_stats()
+        total_vectors = index_stats["total_vector_count"]
+        if total_vectors == 0:
+            return
+        dimension = index_stats["dimension"]
+        query_params = {
+            "filter": {RECORD_ID_LABEL: {"$eq": file_data.identifier}},
+            "vector": [0] * dimension,
+            "top_k": total_vectors,
+        }
+        if namespace := self.upload_config.namespace:
+            query_params["namespace"] = namespace
+        while True:
+            query_results = index.query(**query_params)
+            matches = query_results.get("matches", [])
+            if not matches:
+                break
+            ids = [match["id"] for match in matches]
+            delete_params = {"ids": ids}
+            if namespace := self.upload_config.namespace:
+                delete_params["namespace"] = namespace
+            index.delete(**delete_params)
+        logger.debug(
+            f"deleted any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"from pinecone index"
+        )
+
     @requires_dependencies(["pinecone"], extras="pinecone")
     def upsert_batches_async(self, elements_dict: list[dict]):
         from pinecone.exceptions import PineconeApiException
@@ -208,7 +271,15 @@ class PineconeUploader(Uploader):
             f" index named {self.connection_config.index_name}"
             f" with batch size {self.upload_config.batch_size}"
         )
-
+        # Determine if serverless or pod based index
+        pinecone_client = self.connection_config.get_client()
+        index_description = pinecone_client.describe_index(name=self.connection_config.index_name)
+        if "serverless" in index_description.get("spec"):
+            self.serverless_delete_by_record_id(file_data=file_data)
+        elif "pod" in index_description.get("spec"):
+            self.pod_delete_by_record_id(file_data=file_data)
+        else:
+            raise ValueError(f"unexpected spec type in index description: {index_description}")
         self.upsert_batches_async(elements_dict=elements_dict)
 
 

unstructured_ingest/v2/processes/connectors/qdrant/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+)
+
+from .cloud import CONNECTOR_TYPE as CLOUD_CONNECTOR_TYPE
+from .cloud import qdrant_cloud_destination_entry
+from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
+from .local import qdrant_local_destination_entry
+from .server import CONNECTOR_TYPE as SERVER_CONNECTOR_TYPE
+from .server import qdrant_server_destination_entry
+
+add_destination_entry(destination_type=CLOUD_CONNECTOR_TYPE, entry=qdrant_cloud_destination_entry)
+add_destination_entry(destination_type=SERVER_CONNECTOR_TYPE, entry=qdrant_server_destination_entry)
+add_destination_entry(destination_type=LOCAL_CONNECTOR_TYPE, entry=qdrant_local_destination_entry)

unstructured_ingest/v2/processes/connectors/qdrant/cloud.py

@@ -0,0 +1,59 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.qdrant.qdrant import (
+    QdrantAccessConfig,
+    QdrantConnectionConfig,
+    QdrantUploader,
+    QdrantUploaderConfig,
+    QdrantUploadStager,
+    QdrantUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "qdrant-cloud"
+
+
+class CloudQdrantAccessConfig(QdrantAccessConfig):
+    api_key: str = Field(description="Qdrant API key")
+
+
+class CloudQdrantConnectionConfig(QdrantConnectionConfig):
+    url: str = Field(default=None, description="url of Qdrant Cloud")
+    access_config: Secret[CloudQdrantAccessConfig]
+
+    def get_client_kwargs(self) -> dict:
+        return {
+            "api_key": self.access_config.get_secret_value().api_key,
+            "url": self.url,
+        }
+
+
+class CloudQdrantUploadStagerConfig(QdrantUploadStagerConfig):
+    pass
+
+
+@dataclass
+class CloudQdrantUploadStager(QdrantUploadStager):
+    upload_stager_config: CloudQdrantUploadStagerConfig
+
+
+class CloudQdrantUploaderConfig(QdrantUploaderConfig):
+    pass
+
+
+@dataclass
+class CloudQdrantUploader(QdrantUploader):
+    connection_config: CloudQdrantConnectionConfig
+    upload_config: CloudQdrantUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+qdrant_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=CloudQdrantConnectionConfig,
+    uploader=CloudQdrantUploader,
+    uploader_config=CloudQdrantUploaderConfig,
+    upload_stager=CloudQdrantUploadStager,
+    upload_stager_config=CloudQdrantUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/qdrant/local.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.qdrant.qdrant import (
+    QdrantAccessConfig,
+    QdrantConnectionConfig,
+    QdrantUploader,
+    QdrantUploaderConfig,
+    QdrantUploadStager,
+    QdrantUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "qdrant-local"
+
+
+class LocalQdrantAccessConfig(QdrantAccessConfig):
+    pass
+
+
+class LocalQdrantConnectionConfig(QdrantConnectionConfig):
+    path: str = Field(default=None, description="Persistence path for QdrantLocal.")
+    access_config: Secret[LocalQdrantAccessConfig] = Field(
+        default_factory=LocalQdrantAccessConfig, validate_default=True
+    )
+
+    def get_client_kwargs(self) -> dict:
+        return {"path": self.path}
+
+
+class LocalQdrantUploadStagerConfig(QdrantUploadStagerConfig):
+    pass
+
+
+@dataclass
+class LocalQdrantUploadStager(QdrantUploadStager):
+    upload_stager_config: LocalQdrantUploadStagerConfig
+
+
+class LocalQdrantUploaderConfig(QdrantUploaderConfig):
+    pass
+
+
+@dataclass
+class LocalQdrantUploader(QdrantUploader):
+    connection_config: LocalQdrantConnectionConfig
+    upload_config: LocalQdrantUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+qdrant_local_destination_entry = DestinationRegistryEntry(
+    connection_config=LocalQdrantConnectionConfig,
+    uploader=LocalQdrantUploader,
+    uploader_config=LocalQdrantUploaderConfig,
+    upload_stager=LocalQdrantUploadStager,
+    upload_stager_config=LocalQdrantUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/qdrant/qdrant.py

@@ -0,0 +1,168 @@
+import asyncio
+import json
+import uuid
+from abc import ABC, abstractmethod
+from contextlib import asynccontextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError, WriteError
+from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+
+if TYPE_CHECKING:
+    from qdrant_client import AsyncQdrantClient
+
+
+class QdrantAccessConfig(AccessConfig, ABC):
+    pass
+
+
+class QdrantConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[QdrantAccessConfig] = Field(
+        default_factory=QdrantAccessConfig, validate_default=True, description="Access Config"
+    )
+
+    @abstractmethod
+    def get_client_kwargs(self) -> dict:
+        pass
+
+    @requires_dependencies(["qdrant_client"], extras="qdrant")
+    @asynccontextmanager
+    async def get_client(self) -> AsyncGenerator["AsyncQdrantClient", None]:
+        from qdrant_client.async_qdrant_client import AsyncQdrantClient
+
+        client_kwargs = self.get_client_kwargs()
+        client = AsyncQdrantClient(**client_kwargs)
+        try:
+            yield client
+        finally:
+            await client.close()
+
+
+class QdrantUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class QdrantUploadStager(UploadStager, ABC):
+    upload_stager_config: QdrantUploadStagerConfig = field(
+        default_factory=lambda: QdrantUploadStagerConfig()
+    )
+
+    @staticmethod
+    def conform_dict(data: dict) -> dict:
+        """Prepares dictionary in the format that Chroma requires"""
+        return {
+            "id": str(uuid.uuid4()),
+            "vector": data.pop("embeddings", {}),
+            "payload": {
+                "text": data.pop("text", None),
+                "element_serialized": json.dumps(data),
+                **flatten_dict(
+                    data,
+                    separator="-",
+                    flatten_lists=True,
+                ),
+            },
+        }
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents = json.load(elements_file)
+
+        conformed_elements = [self.conform_dict(data=element) for element in elements_contents]
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+
+        with open(output_path, "w") as output_file:
+            json.dump(conformed_elements, output_file)
+        return output_path
+
+
+class QdrantUploaderConfig(UploaderConfig):
+    collection_name: str = Field(description="Name of the collection.")
+    batch_size: int = Field(default=50, description="Number of records per batch.")
+    num_processes: Optional[int] = Field(
+        default=1,
+        description="Optional limit on number of threads to use for upload.",
+        deprecated=True,
+    )
+
+
+@dataclass
+class QdrantUploader(Uploader, ABC):
+    upload_config: QdrantUploaderConfig
+    connection_config: QdrantConnectionConfig
+
+    @DestinationConnectionError.wrap
+    def precheck(self) -> None:
+        async def check_connection():
+            async with self.connection_config.get_client() as async_client:
+                await async_client.get_collections()
+
+        asyncio.run(check_connection())
+
+    def is_async(self):
+        return True
+
+    async def run_async(
+        self,
+        path: Path,
+        file_data: FileData,
+        **kwargs: Any,
+    ) -> None:
+        with path.open("r") as file:
+            elements: list[dict] = json.load(file)
+
+        logger.debug("Loaded %i elements from %s", len(elements), path)
+
+        batches = list(batch_generator(elements, batch_size=self.upload_config.batch_size))
+        logger.debug(
+            "Elements split into %i batches of size %i.",
+            len(batches),
+            self.upload_config.batch_size,
+        )
+        await asyncio.gather(*[self._upsert_batch(batch) for batch in batches])
+
+    async def _upsert_batch(self, batch: list[dict]) -> None:
+        from qdrant_client import models
+
+        points: list[models.PointStruct] = [models.PointStruct(**item) for item in batch]
+        try:
+            logger.debug(
+                "Upserting %i points to the '%s' collection.",
+                len(points),
+                self.upload_config.collection_name,
+            )
+            async with self.connection_config.get_client() as async_client:
+                await async_client.upsert(
+                    self.upload_config.collection_name, points=points, wait=True
+                )
+        except Exception as api_error:
+            logger.error(
+                "Failed to upsert points to the collection due to the following error %s", api_error
+            )
+
+            raise WriteError(f"Qdrant error: {api_error}") from api_error
+
+        logger.debug("Successfully upsert points to the collection.")

unstructured_ingest/v2/processes/connectors/qdrant/server.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.qdrant.qdrant import (
+    QdrantAccessConfig,
+    QdrantConnectionConfig,
+    QdrantUploader,
+    QdrantUploaderConfig,
+    QdrantUploadStager,
+    QdrantUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "qdrant-server"
+
+
+class ServerQdrantAccessConfig(QdrantAccessConfig):
+    pass
+
+
+class ServerQdrantConnectionConfig(QdrantConnectionConfig):
+    url: str = Field(default=None, description="url of Qdrant server")
+    access_config: Secret[ServerQdrantAccessConfig] = Field(
+        default_factory=ServerQdrantAccessConfig, validate_default=True
+    )
+
+    def get_client_kwargs(self) -> dict:
+        return {
+            "url": self.url,
+        }
+
+
+class ServerQdrantUploadStagerConfig(QdrantUploadStagerConfig):
+    pass
+
+
+@dataclass
+class ServerQdrantUploadStager(QdrantUploadStager):
+    upload_stager_config: ServerQdrantUploadStagerConfig
+
+
+class ServerQdrantUploaderConfig(QdrantUploaderConfig):
+    pass
+
+
+@dataclass
+class ServerQdrantUploader(QdrantUploader):
+    connection_config: ServerQdrantConnectionConfig
+    upload_config: ServerQdrantUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+qdrant_server_destination_entry = DestinationRegistryEntry(
+    connection_config=ServerQdrantConnectionConfig,
+    uploader=ServerQdrantUploader,
+    uploader_config=ServerQdrantUploaderConfig,
+    upload_stager=ServerQdrantUploadStager,
+    upload_stager_config=ServerQdrantUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -21,7 +21,6 @@ from unstructured_ingest.v2.interfaces import (
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
-    download_responses,
 )
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -426,7 +425,7 @@ class SharepointDownloader(Downloader):
             f.write(etree.tostring(document, encoding="unicode", pretty_print=True))
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
-    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         content_type = file_data.additional_metadata.get("sharepoint_content_type")
         if not content_type:
             raise ValueError(
@@ -436,6 +435,8 @@ class SharepointDownloader(Downloader):
             return self.get_document(file_data=file_data)
         elif content_type == SharepointContentType.SITEPAGE.value:
             return self.get_site_page(file_data=file_data)
+        else:
+            raise ValueError(f"content type not recognized: {content_type}")
 
 
 sharepoint_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/slack.py

@@ -16,9 +16,9 @@ from unstructured_ingest.v2.interfaces import (
     ConnectionConfig,
     Downloader,
     DownloaderConfig,
+    DownloadResponse,
     Indexer,
     IndexerConfig,
-    download_responses,
 )
 from unstructured_ingest.v2.interfaces.file_data import (
     FileData,
@@ -161,7 +161,7 @@ class SlackDownloader(Downloader):
     def run(self, file_data, **kwargs):
         raise NotImplementedError
 
-    async def run_async(self, file_data: FileData, **kwargs) -> download_responses:
+    async def run_async(self, file_data: FileData, **kwargs) -> DownloadResponse:
         # NOTE: Indexer should provide source identifiers required to generate the download path
         download_path = self.get_download_path(file_data)
         if download_path is None:

unstructured_ingest/v2/processes/connectors/sql/postgres.py

@@ -98,20 +98,28 @@ class PostgresDownloader(SQLDownloader):
     download_config: PostgresDownloaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["psycopg2"], extras="postgres")
     def query_db(self, file_data: FileData) -> tuple[list[tuple], list[str]]:
+        from psycopg2 import sql
+
         table_name = file_data.additional_metadata["table_name"]
         id_column = file_data.additional_metadata["id_column"]
-        ids = file_data.additional_metadata["ids"]
+        ids = tuple(file_data.additional_metadata["ids"])
+
         with self.connection_config.get_cursor() as cursor:
-            fields = ",".join(self.download_config.fields) if self.download_config.fields else "*"
-            query = "SELECT {fields} FROM {table_name} WHERE {id_column} in ({ids})".format(
+            fields = (
+                sql.SQL(",").join(sql.Identifier(field) for field in self.download_config.fields)
+                if self.download_config.fields
+                else sql.SQL("*")
+            )
+
+            query = sql.SQL("SELECT {fields} FROM {table_name} WHERE {id_column} IN %s").format(
                 fields=fields,
-                table_name=table_name,
-                id_column=id_column,
-                ids=",".join([str(i) for i in ids]),
+                table_name=sql.Identifier(table_name),
+                id_column=sql.Identifier(id_column),
             )
-            logger.debug(f"running query: {query}")
-            cursor.execute(query)
+            logger.debug(f"running query: {cursor.mogrify(query, (ids,))}")
+            cursor.execute(query, (ids,))
             rows = cursor.fetchall()
             columns = [col[0] for col in cursor.description]
             return rows, columns

unstructured_ingest/v2/processes/connectors/sql/snowflake.py

@@ -70,7 +70,9 @@ class SnowflakeConnectionConfig(SQLConnectionConfig):
         connect_kwargs["password"] = self.access_config.get_secret_value().password
         # https://peps.python.org/pep-0249/#paramstyle
         connect_kwargs["paramstyle"] = "qmark"
-        connection = connect(**connect_kwargs)
+        # remove anything that is none
+        active_kwargs = {k: v for k, v in connect_kwargs.items() if v is not None}
+        connection = connect(**active_kwargs)
         try:
             yield connection
         finally:

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -300,10 +300,8 @@ class SQLUploader(Uploader):
 
     def precheck(self) -> None:
         try:
-            connection = self.connection_config.get_connection()
-            cursor = connection.cursor()
-            cursor.execute("SELECT 1;")
-            cursor.close()
+            with self.connection_config.get_cursor() as cursor:
+                cursor.execute("SELECT 1;")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")

unstructured_ingest/v2/processes/partitioner.py

@@ -9,7 +9,7 @@ from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
-from unstructured_ingest.v2.unstructured_api import call_api
+from unstructured_ingest.v2.unstructured_api import call_api_async
 
 
 class PartitionerConfig(BaseModel):
@@ -47,7 +47,11 @@ class PartitionerConfig(BaseModel):
     )
     metadata_exclude: list[str] = Field(
         default_factory=list,
-        description="If set, drop the specified metadata " "fields if they exist.",
+        description="If set, drop the specified metadata fields if they exist.",
+    )
+    element_exclude: list[str] = Field(
+        default_factory=list,
+        description="If set, drop the specified element types, if they exist.",
     )
     metadata_include: list[str] = Field(
         default_factory=list,
@@ -100,6 +104,13 @@ class Partitioner(BaseProcess, ABC):
 
     def postprocess(self, elements: list[dict]) -> list[dict]:
         element_dicts = [e.copy() for e in elements]
+        if self.config.element_exclude:
+            element_dicts = list(
+                filter(
+                    lambda element: element["type"] not in self.config.element_exclude,
+                    element_dicts,
+                )
+            )
         for elem in element_dicts:
             if self.config.metadata_exclude:
                 ex_list = self.config.metadata_exclude
@@ -156,7 +167,7 @@ class Partitioner(BaseProcess, ABC):
         metadata = metadata or {}
         logger.debug(f"partitioning file {filename} with metadata: {metadata}")
 
-        elements = await call_api(
+        elements = await call_api_async(
             server_url=self.config.partition_endpoint,
             api_key=self.config.api_key.get_secret_value(),
             filename=filename,