unstructured-ingest 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/duckdb/base.py

@@ -0,0 +1,99 @@
+import json
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from unstructured_ingest.v2.interfaces import FileData, UploadStager
+
+_COLUMNS = (
+    "id",
+    "element_id",
+    "text",
+    "embeddings",
+    "type",
+    "system",
+    "layout_width",
+    "layout_height",
+    "points",
+    "url",
+    "version",
+    "date_created",
+    "date_modified",
+    "date_processed",
+    "permissions_data",
+    "record_locator",
+    "category_depth",
+    "parent_id",
+    "attached_filename",
+    "filetype",
+    "last_modified",
+    "file_directory",
+    "filename",
+    "languages",
+    "page_number",
+    "links",
+    "page_name",
+    "link_urls",
+    "link_texts",
+    "sent_from",
+    "sent_to",
+    "subject",
+    "section",
+    "header_footer_type",
+    "emphasized_text_contents",
+    "emphasized_text_tags",
+    "text_as_html",
+    "regex_metadata",
+    "detection_class_prob",
+)
+
+# _DATE_COLUMNS = ("date_created", "date_modified", "date_processed", "last_modified")
+
+
+@dataclass
+class BaseDuckDBUploadStager(UploadStager):
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents: list[dict] = json.load(elements_file)
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        output = []
+        for data in elements_contents:
+            metadata: dict[str, Any] = data.pop("metadata", {})
+            data_source = metadata.pop("data_source", {})
+            coordinates = metadata.pop("coordinates", {})
+
+            data.update(metadata)
+            data.update(data_source)
+            data.update(coordinates)
+
+            data["id"] = str(uuid.uuid4())
+
+            # remove extraneous, not supported columns
+            data = {k: v for k, v in data.items() if k in _COLUMNS}
+
+            output.append(data)
+
+        df = pd.DataFrame.from_dict(output)
+
+        for column in filter(
+            lambda x: x in df.columns,
+            ("version", "page_number", "regex_metadata"),
+        ):
+            df[column] = df[column].apply(str)
+
+        with output_path.open("w") as output_file:
+            df.to_json(output_file, orient="records", lines=True)
+        return output_path

unstructured_ingest/v2/processes/connectors/duckdb/duckdb.py

@@ -0,0 +1,118 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    Uploader,
+    UploaderConfig,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+
+if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection as DuckDBConnection
+
+CONNECTOR_TYPE = "duckdb"
+
+
+class DuckDBAccessConfig(AccessConfig):
+    pass
+
+
+class DuckDBConnectionConfig(ConnectionConfig):
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    database: Optional[str] = Field(
+        default=None,
+        description="Database name. Path to the DuckDB .db file. If the file does "
+        "not exist, it will be created at the specified path.",
+    )
+    db_schema: Optional[str] = Field(
+        default="main",
+        description="Schema name. Schema in the database where the elements table is located.",
+    )
+    table: Optional[str] = Field(
+        default="elements",
+        description="Table name. Table name into which the elements data is inserted.",
+    )
+    access_config: Secret[DuckDBAccessConfig] = Field(
+        default=DuckDBAccessConfig(), validate_default=True
+    )
+
+    def __post_init__(self):
+        if self.database is None:
+            raise ValueError(
+                "A DuckDB connection requires a path to a *.db or *.duckdb file "
+                "through the `database` argument"
+            )
+
+
+class DuckDBUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class DuckDBUploadStager(BaseDuckDBUploadStager):
+    upload_stager_config: DuckDBUploadStagerConfig = field(
+        default_factory=lambda: DuckDBUploadStagerConfig()
+    )
+
+
+class DuckDBUploaderConfig(UploaderConfig):
+    batch_size: int = Field(default=50, description="[Not-used] Number of records per batch")
+
+
+@dataclass
+class DuckDBUploader(Uploader):
+    connector_type: str = CONNECTOR_TYPE
+    upload_config: DuckDBUploaderConfig
+    connection_config: DuckDBConnectionConfig
+
+    def precheck(self) -> None:
+        try:
+            cursor = self.connection().cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @property
+    def connection(self) -> Callable[[], "DuckDBConnection"]:
+        return self._make_duckdb_connection
+
+    @requires_dependencies(["duckdb"], extras="duckdb")
+    def _make_duckdb_connection(self) -> "DuckDBConnection":
+        import duckdb
+
+        return duckdb.connect(self.connection_config.database)
+
+    def upload_contents(self, path: Path) -> None:
+        df_elements = pd.read_json(path, orient="records", lines=True)
+        logger.debug(f"uploading {len(df_elements)} entries to {self.connection_config.database} ")
+
+        with self.connection() as conn:
+            conn.query(
+                f"INSERT INTO {self.connection_config.db_schema}.{self.connection_config.table} BY NAME SELECT * FROM df_elements"  # noqa: E501
+            )
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        self.upload_contents(path=path)
+
+
+duckdb_destination_entry = DestinationRegistryEntry(
+    connection_config=DuckDBConnectionConfig,
+    uploader=DuckDBUploader,
+    uploader_config=DuckDBUploaderConfig,
+    upload_stager=DuckDBUploadStager,
+    upload_stager_config=DuckDBUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/duckdb/motherduck.py

@@ -0,0 +1,133 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+import pandas as pd
+from pydantic import Field, Secret
+
+from unstructured_ingest.__version__ import __version__ as unstructured_io_ingest_version
+from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    FileData,
+    Uploader,
+    UploaderConfig,
+    UploadStagerConfig,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+
+if TYPE_CHECKING:
+    from duckdb import DuckDBPyConnection as MotherDuckConnection
+
+CONNECTOR_TYPE = "motherduck"
+
+
+class MotherDuckAccessConfig(AccessConfig):
+    md_token: Optional[str] = Field(default=None, description="MotherDuck token")
+
+
+class MotherDuckConnectionConfig(ConnectionConfig):
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    database: Optional[str] = Field(
+        default=None,
+        description="Database name. Name of the MotherDuck database.",
+    )
+    db_schema: Optional[str] = Field(
+        default="main",
+        description="Schema name. Schema in the database where the elements table is located.",
+    )
+    table: Optional[str] = Field(
+        default="elements",
+        description="Table name. Table name into which the elements data is inserted.",
+    )
+    access_config: Secret[MotherDuckAccessConfig] = Field(
+        default=MotherDuckAccessConfig(), validate_default=True
+    )
+
+    def __post_init__(self):
+        if self.database is None:
+            raise ValueError(
+                "A MotherDuck connection requires a database (string) to be passed "
+                "through the `database` argument"
+            )
+        if self.access_config.md_token is None:
+            raise ValueError(
+                "A MotherDuck connection requires a md_token (MotherDuck token) to be passed "
+                "using MotherDuckAccessConfig through the `access_config` argument"
+            )
+
+
+class MotherDuckUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class MotherDuckUploadStager(BaseDuckDBUploadStager):
+    upload_stager_config: MotherDuckUploadStagerConfig = field(
+        default_factory=lambda: MotherDuckUploadStagerConfig()
+    )
+
+
+class MotherDuckUploaderConfig(UploaderConfig):
+    batch_size: int = Field(default=50, description="[Not-used] Number of records per batch")
+
+
+@dataclass
+class MotherDuckUploader(Uploader):
+    connector_type: str = CONNECTOR_TYPE
+    upload_config: MotherDuckUploaderConfig
+    connection_config: MotherDuckConnectionConfig
+
+    def precheck(self) -> None:
+        try:
+            cursor = self.connection().cursor()
+            cursor.execute("SELECT 1;")
+            cursor.close()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @property
+    def connection(self) -> Callable[[], "MotherDuckConnection"]:
+        return self._make_motherduck_connection
+
+    @requires_dependencies(["duckdb"], extras="duckdb")
+    def _make_motherduck_connection(self) -> "MotherDuckConnection":
+        import duckdb
+
+        access_config = self.connection_config.access_config.get_secret_value()
+        conn = duckdb.connect(
+            f"md:?motherduck_token={access_config.md_token}",
+            config={
+                "custom_user_agent": f"unstructured-io-ingest/{unstructured_io_ingest_version}"
+            },
+        )
+
+        conn.sql(f"USE {self.connection_config.database}")
+
+        return conn
+
+    def upload_contents(self, path: Path) -> None:
+        df_elements = pd.read_json(path, orient="records", lines=True)
+        logger.debug(f"uploading {len(df_elements)} entries to {self.connection_config.database} ")
+
+        with self.connection() as conn:
+            conn.query(
+                f"INSERT INTO {self.connection_config.db_schema}.{self.connection_config.table} BY NAME SELECT * FROM df_elements"  # noqa: E501
+            )
+
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        self.upload_contents(path=path)
+
+
+motherduck_destination_entry = DestinationRegistryEntry(
+    connection_config=MotherDuckConnectionConfig,
+    uploader=MotherDuckUploader,
+    uploader_config=MotherDuckUploaderConfig,
+    upload_stager=MotherDuckUploadStager,
+    upload_stager_config=MotherDuckUploadStagerConfig,
+)

unstructured_ingest/v2/processes/connectors/elasticsearch/elasticsearch.py

@@ -1,7 +1,7 @@
+import collections
 import hashlib
 import json
 import sys
-import uuid
 from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -41,6 +41,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
 )
+from unstructured_ingest.v2.utils import get_enhanced_element_id
 
 if TYPE_CHECKING:
     from elasticsearch import Elasticsearch as ElasticsearchClient
@@ -326,7 +327,7 @@ class ElasticsearchUploadStager(UploadStager):
     def conform_dict(self, data: dict, file_data: FileData) -> dict:
         resp = {
             "_index": self.upload_stager_config.index_name,
-            "_id": str(uuid.uuid4()),
+            "_id": get_enhanced_element_id(element_dict=data, file_data=file_data),
             "_source": {
                 "element_id": data.pop("element_id", None),
                 "embeddings": data.pop("embeddings", None),
@@ -425,7 +426,10 @@ class ElasticsearchUploader(Uploader):
         if failures := delete_resp.get("failures"):
             raise WriteError(f"failed to delete records: {failures}")
 
-    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+    @requires_dependencies(["elasticsearch"], extras="elasticsearch")
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:  # type: ignore
+        from elasticsearch.helpers.errors import BulkIndexError
+
         parallel_bulk = self.load_parallel_bulk()
         with path.open("r") as file:
             elements_dict = json.load(file)
@@ -449,18 +453,33 @@ class ElasticsearchUploader(Uploader):
             for batch in generator_batching_wbytes(
                 elements_dict, batch_size_limit_bytes=self.upload_config.batch_size_bytes
             ):
-                for success, info in parallel_bulk(
-                    client=client,
-                    actions=batch,
-                    thread_count=self.upload_config.num_threads,
-                ):
-                    if not success:
-                        logger.error(
-                            "upload failed for a batch in "
-                            f"{(self.__class__.__name__).replace('Uploader', '')} "
-                            "destination connector:",
-                            info,
-                        )
+                try:
+                    iterator = parallel_bulk(
+                        client=client,
+                        actions=batch,
+                        thread_count=self.upload_config.num_threads,
+                    )
+                    collections.deque(iterator, maxlen=0)
+                except BulkIndexError as e:
+                    sanitized_errors = [
+                        self._sanitize_bulk_index_error(error) for error in e.errors
+                    ]
+                    logger.error(
+                        f"Batch upload failed - {e} - with following errors: {sanitized_errors}"
+                    )
+                    raise e
+                except Exception as e:
+                    logger.error(f"Batch upload failed - {e}")
+                    raise e
+
+    def _sanitize_bulk_index_error(self, error: dict[str, dict]) -> dict:
+        """Remove data uploaded to index from the log, leave only error information.
+
+        Error structure is `{<operation-type>: {..., "data": <uploaded-object>}}`
+        """
+        for error_data in error.values():
+            error_data.pop("data", None)
+        return error
 
 
 elasticsearch_source_entry = SourceRegistryEntry(

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -110,10 +110,14 @@ class AzureIndexer(FsspecIndexer):
     def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
         path = file_data["name"]
         date_created = (
-            file_data.get("creation_time").timestamp() if "creation_time" in file_data else None
+            str(file_data.get("creation_time").timestamp())
+            if "creation_time" in file_data
+            else None
         )
         date_modified = (
-            file_data.get("last_modified").timestamp() if "last_modified" in file_data else None
+            str(file_data.get("last_modified").timestamp())
+            if "last_modified" in file_data
+            else None
         )
 
         file_size = file_data.get("size") if "size" in file_data else None

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -3,10 +3,11 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
-from typing import Any, Generator, Optional
+from typing import Annotated, Any, Generator, Optional
 
 from dateutil import parser
 from pydantic import Field, Secret
+from pydantic.functional_validators import BeforeValidator
 
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, FileDataSourceMetadata
@@ -23,7 +24,9 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecIndexerConfig,
     FsspecUploader,
     FsspecUploaderConfig,
+    SourceConnectionError,
 )
+from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
 
 CONNECTOR_TYPE = "box"
 
@@ -33,26 +36,35 @@ class BoxIndexerConfig(FsspecIndexerConfig):
 
 
 class BoxAccessConfig(FsspecAccessConfig):
-    box_app_config: Optional[str] = Field(
-        default=None, description="Path to Box app credentials as json file."
+    box_app_config: Annotated[dict, BeforeValidator(conform_string_to_dict)] = Field(
+        description="Box app credentials as a JSON string."
     )
 
 
 class BoxConnectionConfig(FsspecConnectionConfig):
     supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
-    access_config: Secret[BoxAccessConfig] = Field(default=BoxAccessConfig(), validate_default=True)
+    access_config: Secret[BoxAccessConfig]
     connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
-        # Return access_kwargs with oauth. The oauth object cannot be stored directly in the config
-        # because it is not serializable.
         from boxsdk import JWTAuth
 
         ac = self.access_config.get_secret_value()
+        settings_dict = ac.box_app_config
+
+        # Create and authenticate the JWTAuth object
+        oauth = JWTAuth.from_settings_dictionary(settings_dict)
+        try:
+            oauth.authenticate_instance()
+        except Exception as e:
+            raise SourceConnectionError(f"Failed to authenticate with Box: {e}")
+
+        if not oauth.access_token:
+            raise SourceConnectionError("Authentication failed: No access token generated.")
+
+        # Prepare the access configuration with the authenticated oauth
         access_kwargs_with_oauth: dict[str, Any] = {
-            "oauth": JWTAuth.from_settings_file(
-                ac.box_app_config,
-            ),
+            "oauth": oauth,
         }
         access_config: dict[str, Any] = ac.model_dump()
         access_config.pop("box_app_config", None)
@@ -80,9 +92,9 @@ class BoxIndexer(FsspecIndexer):
         date_created = None
         date_modified = None
         if modified_at_str := file_data.get("modified_at"):
-            date_modified = parser.parse(modified_at_str).timestamp()
+            date_modified = str(parser.parse(modified_at_str).timestamp())
         if created_at_str := file_data.get("created_at"):
-            date_created = parser.parse(created_at_str).timestamp()
+            date_created = str(parser.parse(created_at_str).timestamp())
 
         file_size = file_data.get("size") if "size" in file_data else None
 

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -297,7 +297,7 @@ class FsspecUploader(Uploader):
                 **self.connection_config.get_access_config(),
             )
             upload_path = Path(self.upload_config.path_without_protocol) / "_empty"
-            fs.write_bytes(path=str(upload_path), value=b"")
+            fs.write_bytes(path=upload_path.as_posix(), value=b"")
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
@@ -314,11 +314,11 @@ class FsspecUploader(Uploader):
         path_str = str(path.resolve())
         upload_path = self.get_upload_path(file_data=file_data)
         logger.debug(f"writing local file {path_str} to {upload_path}")
-        self.fs.upload(lpath=path_str, rpath=str(upload_path))
+        self.fs.upload(lpath=path_str, rpath=upload_path.as_posix())
 
     async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
         upload_path = self.get_upload_path(file_data=file_data)
         path_str = str(path.resolve())
         # Odd that fsspec doesn't run exists() as async even when client support async
         logger.debug(f"writing local file {path_str} to {upload_path}")
-        self.fs.upload(lpath=path_str, rpath=str(upload_path))
+        self.fs.upload(lpath=path_str, rpath=upload_path.as_posix())

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -113,9 +113,9 @@ class GcsIndexer(FsspecIndexer):
         date_created = None
         date_modified = None
         if modified_at_str := file_data.get("updated"):
-            date_modified = parser.parse(modified_at_str).timestamp()
+            date_modified = str(parser.parse(modified_at_str).timestamp())
         if created_at_str := file_data.get("timeCreated"):
-            date_created = parser.parse(created_at_str).timestamp()
+            date_created = str(parser.parse(created_at_str).timestamp())
 
         file_size = file_data.get("size") if "size" in file_data else None
 

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -30,7 +30,6 @@ CONNECTOR_TYPE = "sftp"
 
 
 class SftpIndexerConfig(FsspecIndexerConfig):
-
     def model_post_init(self, __context: Any) -> None:
         super().model_post_init(__context)
         _, ext = os.path.splitext(self.remote_url)
@@ -99,8 +98,8 @@ class SftpIndexer(FsspecIndexer):
 
     def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
         path = file_data["name"]
-        date_created = file_data.get("time").timestamp() if "time" in file_data else None
-        date_modified = file_data.get("mtime").timestamp() if "mtime" in file_data else None
+        date_created = str(file_data.get("time").timestamp()) if "time" in file_data else None
+        date_modified = str(file_data.get("mtime").timestamp()) if "mtime" in file_data else None
 
         file_size = file_data.get("size") if "size" in file_data else None
 

unstructured_ingest/v2/processes/connectors/kafka/cloud.py

@@ -1,6 +1,6 @@
 import socket
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 from pydantic import Field, Secret, SecretStr
 
@@ -26,10 +26,10 @@ CONNECTOR_TYPE = "kafka-cloud"
 
 
 class CloudKafkaAccessConfig(KafkaAccessConfig):
-    api_key: Optional[SecretStr] = Field(
-        description="Kafka API key to connect at the server", alias="kafka_api_key", default=None
+    kafka_api_key: SecretStr = Field(
+        description="Kafka API key to connect at the server", default=None
     )
-    secret: Optional[SecretStr] = Field(description="", default=None)
+    secret: SecretStr = Field(description="", default=None)
 
 
 class CloudKafkaConnectionConfig(KafkaConnectionConfig):
@@ -43,11 +43,11 @@ class CloudKafkaConnectionConfig(KafkaConnectionConfig):
         conf = {
             "bootstrap.servers": f"{bootstrap}:{port}",
             "client.id": socket.gethostname(),
-            "group.id": "default_group_id",
+            "group.id": self.group_id,
             "enable.auto.commit": "false",
             "auto.offset.reset": "earliest",
-            "sasl.username": access_config.api_key,
-            "sasl.password": access_config.secret,
+            "sasl.username": access_config.kafka_api_key.get_secret_value(),
+            "sasl.password": access_config.secret.get_secret_value(),
             "sasl.mechanism": "PLAIN",
             "security.protocol": "SASL_SSL",
         }
@@ -61,7 +61,7 @@ class CloudKafkaConnectionConfig(KafkaConnectionConfig):
 
         conf = {
             "bootstrap.servers": f"{bootstrap}:{port}",
-            "sasl.username": access_config.api_key,
+            "sasl.username": access_config.kafka_api_key,
             "sasl.password": access_config.secret,
             "sasl.mechanism": "PLAIN",
             "security.protocol": "SASL_SSL",

unstructured_ingest/v2/processes/connectors/kafka/kafka.py

@@ -43,6 +43,11 @@ class KafkaConnectionConfig(ConnectionConfig, ABC):
     access_config: Secret[KafkaAccessConfig]
     bootstrap_server: str
     port: int
+    group_id: str = Field(
+        description="A consumer group is a way to allow a pool of consumers "
+        "to divide the consumption of data over topics and partitions.",
+        default="default_group_id",
+    )
 
     @abstractmethod
     def get_consumer_configuration(self) -> dict:
@@ -75,7 +80,7 @@ class KafkaConnectionConfig(ConnectionConfig, ABC):
 class KafkaIndexerConfig(IndexerConfig):
     topic: str = Field(description="which topic to consume from")
     num_messages_to_consume: Optional[int] = 100
-    timeout: Optional[float] = Field(default=1.0, description="polling timeout")
+    timeout: Optional[float] = Field(default=3.0, description="polling timeout", ge=3.0)
 
     def update_consumer(self, consumer: "Consumer") -> None:
         consumer.subscribe([self.topic])
@@ -157,7 +162,9 @@ class KafkaIndexer(Indexer, ABC):
     def precheck(self):
         try:
             with self.get_consumer() as consumer:
-                cluster_meta = consumer.list_topics(timeout=self.index_config.timeout)
+                # timeout needs at least 3 secs, more info:
+                # https://forum.confluent.io/t/kafkacat-connect-failure-to-confcloud-ssl/2513
+                cluster_meta = consumer.list_topics(timeout=5)
                 current_topics = [
                     topic for topic in cluster_meta.topics if topic != "__consumer_offsets"
                 ]

unstructured_ingest/v2/processes/connectors/kafka/local.py

@@ -39,7 +39,7 @@ class LocalKafkaConnectionConfig(KafkaConnectionConfig):
 
         conf = {
             "bootstrap.servers": f"{bootstrap}:{port}",
-            "group.id": "default_group_id",
+            "group.id": self.group_id,
             "enable.auto.commit": "false",
             "auto.offset.reset": "earliest",
         }