unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/interfaces.py

@@ -11,19 +11,20 @@ from abc import ABC, abstractmethod
 from dataclasses import InitVar, dataclass, field
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Optional, Type, TypeVar
+from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar
 
 from dataclasses_json import DataClassJsonMixin
 from dataclasses_json.core import Json, _decode_dataclass
-from unstructured.documents.elements import DataSourceMetadata
-from unstructured.embed.interfaces import BaseEmbeddingEncoder, Element
-from unstructured.partition.api import partition_via_api
-from unstructured.staging.base import elements_to_dicts, flatten_dict
 
 from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
 from unstructured_ingest.enhanced_dataclass.core import _asdict
 from unstructured_ingest.error import PartitionError, SourceConnectionError
 from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.data_prep import flatten_dict
+
+if TYPE_CHECKING:
+    from unstructured.documents.elements import Element
+    from unstructured.embed.interfaces import BaseEmbeddingEncoder
 
 A = TypeVar("A", bound="DataClassJsonMixin")
 
@@ -195,7 +196,7 @@ class EmbeddingConfig(BaseConfig):
     aws_secret_access_key: Optional[str] = None
     aws_region: Optional[str] = None
 
-    def get_embedder(self) -> BaseEmbeddingEncoder:
+    def get_embedder(self) -> "BaseEmbeddingEncoder":
         kwargs: dict[str, Any] = {}
         if self.api_key:
             kwargs["api_key"] = self.api_key
@@ -551,7 +552,8 @@ class BaseSingleIngestDoc(BaseIngestDoc, IngestDocJsonMixin, ABC):
         self,
         partition_config: PartitionConfig,
         **partition_kwargs,
-    ) -> list[Element]:
+    ) -> list["Element"]:
+        from unstructured.documents.elements import DataSourceMetadata
         from unstructured.partition.auto import partition
 
         if not partition_config.partition_by_api:
@@ -570,6 +572,8 @@ class BaseSingleIngestDoc(BaseIngestDoc, IngestDocJsonMixin, ABC):
                 **partition_kwargs,
             )
         else:
+            from unstructured.partition.api import partition_via_api
+
             endpoint = partition_config.partition_endpoint
 
             logger.debug(f"Using remote partition ({endpoint})")
@@ -595,7 +599,7 @@ class BaseSingleIngestDoc(BaseIngestDoc, IngestDocJsonMixin, ABC):
         logger.info(f"Processing {self.filename}")
 
         elements = self.partition_file(partition_config=partition_config, **partition_kwargs)
-        element_dicts = elements_to_dicts(elements)
+        element_dicts = [e.to_dict() for e in elements]
 
         self.isd_elems_no_filename: list[dict[str, Any]] = []
         for elem in element_dicts:
@@ -736,7 +740,7 @@ class BaseDestinationConnector(BaseConnector, ABC):
         elements_dict_normalized = [self.normalize_dict(element_dict=d) for d in elements_dict]
         return self.write_dict(*args, elements_dict=elements_dict_normalized, **kwargs)
 
-    def write_elements(self, elements: list[Element], *args, **kwargs) -> None:
+    def write_elements(self, elements: list["Element"], *args, **kwargs) -> None:
         elements_dict = [e.to_dict() for e in elements]
         self.modify_and_write_dict(*args, elements_dict=elements_dict, **kwargs)
 

unstructured_ingest/pipeline/interfaces.py

@@ -8,11 +8,9 @@ from dataclasses import dataclass, field
 from multiprocessing.managers import DictProxy
 from pathlib import Path
 
-import backoff
 from dataclasses_json import DataClassJsonMixin
 
 from unstructured_ingest.error import SourceConnectionNetworkError
-from unstructured_ingest.ingest_backoff import RetryHandler
 from unstructured_ingest.interfaces import (
     BaseDestinationConnector,
     BaseSourceConnector,
@@ -23,6 +21,9 @@ from unstructured_ingest.interfaces import (
 )
 from unstructured_ingest.logger import ingest_log_streaming_init, logger
 
+if t.TYPE_CHECKING:
+    from unstructured_ingest.ingest_backoff import RetryHandler
+
 
 @dataclass
 class PipelineContext(ProcessorConfig):
@@ -147,8 +148,12 @@ class SourceNode(PipelineNode):
     retry_strategy_config: t.Optional[RetryStrategyConfig] = None
 
     @property
-    def retry_strategy(self) -> t.Optional[RetryHandler]:
+    def retry_strategy(self) -> t.Optional["RetryHandler"]:
         if retry_strategy_config := self.retry_strategy_config:
+            import backoff
+
+            from unstructured_ingest.ingest_backoff import RetryHandler
+
             return RetryHandler(
                 backoff.expo,
                 SourceConnectionNetworkError,

unstructured_ingest/pipeline/reformat/chunking.py

@@ -5,16 +5,15 @@ import json
 import os.path
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
-
-from unstructured.chunking import dispatch
-from unstructured.documents.elements import Element, assign_and_map_hash_ids
-from unstructured.partition.api import partition_via_api
-from unstructured.staging.base import elements_from_json, elements_to_dicts
+from typing import TYPE_CHECKING, Optional
 
 from unstructured_ingest.interfaces import ChunkingConfig, PartitionConfig
 from unstructured_ingest.logger import logger
 from unstructured_ingest.pipeline.interfaces import ReformatNode
+from unstructured_ingest.utils.chunking import assign_and_map_hash_ids
+
+if TYPE_CHECKING:
+    from unstructured.documents.elements import Element
 
 
 @dataclass
@@ -69,9 +68,9 @@ class Chunker(ReformatNode):
                 logger.info(f"chunking_strategy is None, skipping chunking for {filename_ext}")
                 return
 
-            assign_and_map_hash_ids(chunked_elements)
+            element_dicts = [e.to_dict() for e in chunked_elements]
+            assign_and_map_hash_ids(elements=element_dicts)
 
-            element_dicts = elements_to_dicts(chunked_elements)
             with open(json_path, "w", encoding="utf8") as output_f:
                 logger.info(f"writing chunking content to {json_path}")
                 json.dump(element_dicts, output_f, ensure_ascii=False, indent=2)
@@ -86,13 +85,16 @@ class Chunker(ReformatNode):
     def get_path(self) -> Path:
         return (Path(self.pipeline_context.work_dir) / "chunked").resolve()
 
-    def chunk(self, elements_json_file: str) -> Optional[list[Element]]:
+    def chunk(self, elements_json_file: str) -> Optional[list["Element"]]:
         """Called by Chunker.run() to properly execute the defined chunking_strategy."""
         # -- No chunking_strategy means no chunking --
         if self.chunking_config.chunking_strategy is None:
             return
         # -- Chunk locally for open-source chunking strategies, even when partitioning remotely --
         if self.chunking_config.chunking_strategy in ("basic", "by_title"):
+            from unstructured.chunking import dispatch
+            from unstructured.staging.base import elements_from_json
+
             return dispatch.chunk(
                 elements=elements_from_json(filename=elements_json_file),
                 chunking_strategy=self.chunking_config.chunking_strategy,
@@ -106,6 +108,8 @@ class Chunker(ReformatNode):
             )
         # -- Chunk remotely --
         if self.partition_config.partition_by_api:
+            from unstructured.partition.api import partition_via_api
+
             return partition_via_api(
                 filename=elements_json_file,
                 # -- (jennings) If api_key or api_url are None, partition_via_api will raise an

unstructured_ingest/pipeline/reformat/embedding.py

@@ -5,8 +5,6 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional
 
-from unstructured.staging.base import elements_from_json, elements_to_dicts
-
 from unstructured_ingest.interfaces import (
     EmbeddingConfig,
 )
@@ -29,6 +27,8 @@ class Embedder(ReformatNode):
         return hashlib.sha256(json.dumps(hash_dict, sort_keys=True).encode()).hexdigest()[:32]
 
     def run(self, elements_json: str) -> Optional[str]:
+        from unstructured.staging.base import elements_from_json
+
         try:
             elements_json_filename = os.path.basename(elements_json)
             filename_ext = os.path.basename(elements_json_filename)
@@ -51,7 +51,7 @@ class Embedder(ReformatNode):
             elements = elements_from_json(filename=elements_json)
             embedder = self.embedder_config.get_embedder()
             embedded_elements = embedder.embed_documents(elements=elements)
-            element_dicts = elements_to_dicts(embedded_elements)
+            element_dicts = [e.to_dict() for e in embedded_elements]
             with open(json_path, "w", encoding="utf8") as output_f:
                 logger.info(f"writing embeddings content to {json_path}")
                 json.dump(element_dicts, output_f, ensure_ascii=False, indent=2)

unstructured_ingest/runner/__init__.py

@@ -2,7 +2,7 @@ import typing as t
 from typing import Type
 
 from .airtable import AirtableRunner
-from .astra import AstraRunner
+from .astradb import AstraDBRunner
 from .base_runner import Runner
 from .biomed import BiomedRunner
 from .confluence import ConfluenceRunner
@@ -36,7 +36,7 @@ from .wikipedia import WikipediaRunner
 
 runner_map: t.Dict[str, Type[Runner]] = {
     "airtable": AirtableRunner,
-    "astra": AstraRunner,
+    "astradb": AstraDBRunner,
     "azure": AzureRunner,
     "biomed": BiomedRunner,
     "box": BoxRunner,

unstructured_ingest/runner/{astra.py → astradb.py}

@@ -8,27 +8,27 @@ from unstructured_ingest.runner.base_runner import Runner
 from unstructured_ingest.runner.utils import update_download_dir_hash
 
 if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.astra import SimpleAstraConfig
+    from unstructured_ingest.connector.astradb import SimpleAstraDBConfig
 
 
 @dataclass
-class AstraRunner(Runner):
-    connector_config: "SimpleAstraConfig"
+class AstraDBRunner(Runner):
+    connector_config: "SimpleAstraDBConfig"
 
     def update_read_config(self):
         hashed_dir_name = hashlib.sha256(
             str(self.connector_config.access_config.api_endpoint).encode("utf-8"),
         )
         self.read_config.download_dir = update_download_dir_hash(
-            connector_name="astra",
+            connector_name="astradb",
             read_config=self.read_config,
             hashed_dir_name=hashed_dir_name,
             logger=logger,
         )
 
     def get_source_connector_cls(self) -> t.Type[BaseSourceConnector]:
-        from unstructured_ingest.connector.astra import (
-            AstraSourceConnector,
+        from unstructured_ingest.connector.astradb import (
+            AstraDBSourceConnector,
         )
 
-        return AstraSourceConnector
+        return AstraDBSourceConnector

unstructured_ingest/runner/writers/__init__.py

@@ -1,6 +1,6 @@
 import typing as t
 
-from .astra import AstraWriter
+from .astradb import AstraDBWriter
 from .azure_cognitive_search import AzureCognitiveSearchWriter
 from .base_writer import Writer
 from .chroma import ChromaWriter
@@ -23,7 +23,7 @@ from .vectara import VectaraWriter
 from .weaviate import WeaviateWriter
 
 writer_map: t.Dict[str, t.Type[Writer]] = {
-    "astra": AstraWriter,
+    "astradb": AstraDBWriter,
     "azure": AzureWriter,
     "azure_cognitive_search": AzureCognitiveSearchWriter,
     "box": BoxWriter,

unstructured_ingest/runner/writers/{astra.py → astradb.py}

@@ -6,17 +6,17 @@ from unstructured_ingest.interfaces import BaseDestinationConnector
 from unstructured_ingest.runner.writers.base_writer import Writer
 
 if t.TYPE_CHECKING:
-    from unstructured_ingest.connector.astra import AstraWriteConfig, SimpleAstraConfig
+    from unstructured_ingest.connector.astradb import AstraDBWriteConfig, SimpleAstraDBConfig
 
 
 @dataclass
-class AstraWriter(Writer, EnhancedDataClassJsonMixin):
-    write_config: "AstraWriteConfig"
-    connector_config: "SimpleAstraConfig"
+class AstraDBWriter(Writer, EnhancedDataClassJsonMixin):
+    write_config: "AstraDBWriteConfig"
+    connector_config: "SimpleAstraDBConfig"
 
     def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
-        from unstructured_ingest.connector.astra import (
-            AstraDestinationConnector,
+        from unstructured_ingest.connector.astradb import (
+            AstraDBDestinationConnector,
         )
 
-        return AstraDestinationConnector
+        return AstraDBDestinationConnector

unstructured_ingest/utils/chunking.py

@@ -0,0 +1,45 @@
+import hashlib
+from itertools import groupby
+
+
+def id_to_hash(element: dict, sequence_number: int) -> str:
+    """Calculates and assigns a deterministic hash as an ID.
+
+    The hash ID is based on element's text, sequence number on page,
+    page number and its filename.
+
+    Args:
+        sequence_number: index on page
+
+    Returns: new ID value
+    """
+    filename = element["metadata"].get("filename")
+    text = element["text"]
+    page_number = element["metadata"].get("page_number")
+    data = f"{filename}{text}{page_number}{sequence_number}"
+    element["element_id"] = hashlib.sha256(data.encode()).hexdigest()[:32]
+    return element["element_id"]
+
+
+def assign_and_map_hash_ids(elements: list[dict]) -> list[dict]:
+    # -- generate sequence number for each element on a page --
+    elements = elements.copy()
+    page_numbers = [e["metadata"].get("page_number") for e in elements]
+    page_seq_pairs = [
+        seq_on_page for page, group in groupby(page_numbers) for seq_on_page, _ in enumerate(group)
+    ]
+
+    # -- assign hash IDs to elements --
+    old_to_new_mapping = {
+        element["element_id"]: id_to_hash(element=element, sequence_number=seq_on_page_counter)
+        for element, seq_on_page_counter in zip(elements, page_seq_pairs)
+    }
+
+    # -- map old parent IDs to new ones --
+    for e in elements:
+        parent_id = e["metadata"].get("parent_id")
+        if not parent_id:
+            continue
+        e["metadata"]["parent_id"] = old_to_new_mapping[parent_id]
+
+    return elements

unstructured_ingest/utils/dep_check.py

@@ -33,7 +33,7 @@ def requires_dependencies(
                 raise ImportError(
                     f"Following dependencies are missing: {', '.join(missing_deps)}. "
                     + (
-                        f"""Please install them using `pip install "unstructured[{extras}]"`."""
+                        f"""Please install them using `pip install "unstructured-ingest[{extras}]"`."""  # noqa: E501
                         if extras
                         else f"Please install them using `pip install {' '.join(missing_deps)}`."
                     ),

unstructured_ingest/utils/google_filetype.py

@@ -0,0 +1,9 @@
+GOOGLE_DRIVE_EXPORT_TYPES = {
+    "application/vnd.google-apps.document": "application/"
+    "vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.google-apps.spreadsheet": "application/"
+    "vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.google-apps.presentation": "application/"
+    "vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.google-apps.photo": "image/jpeg",
+}

unstructured_ingest/v2/cli/base/cmd.py

@@ -1,13 +1,15 @@
 import inspect
 from abc import ABC, abstractmethod
+from collections import Counter
 from dataclasses import dataclass, field, fields
 from typing import Any, Optional, Type, TypeVar
 
 import click
+from pydantic import BaseModel
 
 from unstructured_ingest.v2.cli.base.importer import import_from_string
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import extract_config
+from unstructured_ingest.v2.cli.utils.click import extract_config
+from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model, post_check
 from unstructured_ingest.v2.interfaces import ProcessorConfig
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.pipeline import Pipeline
@@ -15,6 +17,7 @@ from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
 from unstructured_ingest.v2.processes.connector_registry import (
     DownloaderT,
     IndexerT,
+    RegistryEntry,
     UploaderT,
     UploadStager,
     UploadStagerConfig,
@@ -24,6 +27,7 @@ from unstructured_ingest.v2.processes.connector_registry import (
 )
 from unstructured_ingest.v2.processes.connectors.local import LocalUploader, LocalUploaderConfig
 from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.v2.processes.filter import Filterer, FiltererConfig
 from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
 
 CommandT = TypeVar("CommandT", bound=click.Command)
@@ -32,7 +36,52 @@ CommandT = TypeVar("CommandT", bound=click.Command)
 @dataclass
 class BaseCmd(ABC):
     cmd_name: str
-    default_configs: list[Type[CliConfig]] = field(default_factory=list)
+    registry_entry: RegistryEntry
+    default_configs: list[Type[BaseModel]] = field(default_factory=list)
+
+    @abstractmethod
+    def get_registry_options(self):
+        pass
+
+    def get_default_options(self) -> list[click.Option]:
+        options = []
+        for extra in self.default_configs:
+            options.extend(options_from_base_model(model=extra))
+        return options
+
+    @classmethod
+    def consolidate_options(cls, options: list[click.Option]) -> list[click.Option]:
+        option_names = [option.name for option in options]
+        duplicate_names = [name for name, count in Counter(option_names).items() if count > 1]
+        if not duplicate_names:
+            return options
+        consolidated_options = []
+        current_names = []
+        for option in options:
+            if option.name not in current_names:
+                current_names.append(option.name)
+                consolidated_options.append(option)
+                continue
+            existing_option = next(o for o in consolidated_options if o.name == option.name)
+            if existing_option.__dict__ == option.__dict__:
+                continue
+            option_diff = cls.get_options_diff(o1=option, o2=existing_option)
+            raise ValueError(
+                "Conflicting duplicate {} option defined: {}".format(
+                    option.name, " | ".join([f"{d[0]}: {d[1]}" for d in option_diff])
+                )
+            )
+        return consolidated_options
+
+    @staticmethod
+    def get_options_diff(o1: click.Option, o2: click.Option):
+        o1_dict = o1.__dict__
+        o2_dict = o2.__dict__
+        for d in [o1_dict, o2_dict]:
+            d["opts"] = ",".join(d["opts"])
+            d["secondary_opts"] = ",".join(d["secondary_opts"])
+        option_diff = set(o1_dict.items()) ^ set(o2_dict.items())
+        return option_diff
 
     @property
     def cmd_name_key(self):
@@ -46,15 +95,11 @@ class BaseCmd(ABC):
     def cmd(self, ctx: click.Context, **options) -> None:
         pass
 
-    def add_options(self, cmd: CommandT, extras: list[Type[CliConfig]]) -> CommandT:
-        configs = self.default_configs
-        # make sure what's unique to this cmd appears first
-        extras.extend(configs)
-        for config in extras:
-            try:
-                config.add_cli_options(cmd=cmd)
-            except ValueError as e:
-                raise ValueError(f"failed to set configs from {config.__name__}: {e}")
+    def add_options(self, cmd: CommandT) -> CommandT:
+        options = self.get_registry_options()
+        options.extend(self.get_default_options())
+        post_check(options)
+        cmd.params.extend(options)
         return cmd
 
     def get_pipline(
@@ -75,6 +120,8 @@ class BaseCmd(ABC):
         }
         if chunker := self.get_chunker(options=source_options):
             pipeline_kwargs["chunker"] = chunker
+        if filterer := self.get_filterer(options=source_options):
+            pipeline_kwargs["filterer"] = filterer
         if embedder := self.get_embeder(options=source_options):
             pipeline_kwargs["embedder"] = embedder
         if dest:
@@ -105,6 +152,13 @@ class BaseCmd(ABC):
             return None
         return Chunker(config=chunker_config)
 
+    @staticmethod
+    def get_filterer(options: dict[str, Any]) -> Optional[Filterer]:
+        filterer_configs = extract_config(flat_data=options, config=FiltererConfig)
+        if not filterer_configs.dict():
+            return None
+        return Filterer(config=filterer_configs)
+
     @staticmethod
     def get_embeder(options: dict[str, Any]) -> Optional[Embedder]:
         embedder_config = extract_config(flat_data=options, config=EmbedderConfig)

unstructured_ingest/v2/cli/base/dest.py

@@ -1,20 +1,34 @@
 import logging
 from dataclasses import dataclass
-from typing import Optional, Type
 
 import click
 
 from unstructured_ingest.v2.cli.base.cmd import BaseCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import Dict, conform_click_options
+from unstructured_ingest.v2.cli.utils.click import Dict, conform_click_options
+from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
 
 
 @dataclass
 class DestCmd(BaseCmd):
-    connection_config: Optional[Type[CliConfig]] = None
-    uploader_config: Optional[Type[CliConfig]] = None
-    upload_stager_config: Optional[Type[CliConfig]] = None
+    registry_entry: DestinationRegistryEntry
+
+    def get_registry_options(self):
+        options = []
+        configs = [
+            config
+            for config in [
+                self.registry_entry.uploader_config,
+                self.registry_entry.upload_stager_config,
+                self.registry_entry.connection_config,
+            ]
+            if config
+        ]
+        for config in configs:
+            options.extend(options_from_base_model(model=config))
+        options = self.consolidate_options(options=options)
+        return options
 
     def cmd(self, ctx: click.Context, **options) -> None:
         logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
@@ -47,12 +61,7 @@ class DestCmd(BaseCmd):
         cmd.name = self.cli_cmd_name
         cmd.short_help = "v2"
         cmd.invoke_without_command = True
-        extras = [
-            x
-            for x in [self.uploader_config, self.upload_stager_config, self.connection_config]
-            if x
-        ]
-        self.add_options(cmd, extras=extras)
+        self.add_options(cmd)
         cmd.params.append(
             click.Option(
                 ["--custom-stager"],

unstructured_ingest/v2/cli/base/src.py

@@ -1,35 +1,52 @@
 import logging
 from dataclasses import dataclass, field
-from typing import Optional, Type
 
 import click
+from pydantic import BaseModel
 
 from unstructured_ingest.v2.cli.base.cmd import BaseCmd
-from unstructured_ingest.v2.cli.configs import (
-    ChunkerCliConfig,
-    EmbedderCliConfig,
-    PartitionerCliConfig,
-    ProcessorCliConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import Group, conform_click_options
+from unstructured_ingest.v2.cli.utils.click import Group, conform_click_options
+from unstructured_ingest.v2.cli.utils.model_conversion import options_from_base_model
+from unstructured_ingest.v2.interfaces import ProcessorConfig
 from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes import (
+    ChunkerConfig,
+    EmbedderConfig,
+    FiltererConfig,
+    PartitionerConfig,
+)
+from unstructured_ingest.v2.processes.connector_registry import SourceRegistryEntry
 
 
 @dataclass
 class SrcCmd(BaseCmd):
-    indexer_config: Optional[Type[CliConfig]] = None
-    downloader_config: Optional[Type[CliConfig]] = None
-    connection_config: Optional[Type[CliConfig]] = None
-    default_configs: list[CliConfig] = field(
+    registry_entry: SourceRegistryEntry
+    default_configs: list[BaseModel] = field(
         default_factory=lambda: [
-            ProcessorCliConfig,
-            PartitionerCliConfig,
-            EmbedderCliConfig,
-            ChunkerCliConfig,
+            ProcessorConfig,
+            PartitionerConfig,
+            EmbedderConfig,
+            FiltererConfig,
+            ChunkerConfig,
         ]
     )
 
+    def get_registry_options(self):
+        options = []
+        configs = [
+            config
+            for config in [
+                self.registry_entry.connection_config,
+                self.registry_entry.indexer_config,
+                self.registry_entry.downloader_config,
+            ]
+            if config
+        ]
+        for config in configs:
+            options.extend(options_from_base_model(model=config))
+        options = self.consolidate_options(options=options)
+        return options
+
     def cmd(self, ctx: click.Context, **options) -> None:
         if ctx.invoked_subcommand:
             return
@@ -53,10 +70,7 @@ class SrcCmd(BaseCmd):
         cmd.name = self.cli_cmd_name
         cmd.short_help = "v2"
         cmd.invoke_without_command = True
-        extras = [
-            x for x in [self.indexer_config, self.downloader_config, self.connection_config] if x
-        ]
-        self.add_options(cmd, extras=extras)
+        self.add_options(cmd)
 
         # TODO remove after v1 no longer supported
         cmd.params.append(

unstructured_ingest/v2/cli/cmds.py

@@ -0,0 +1,14 @@
+import click
+
+from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
+from unstructured_ingest.v2.processes.connector_registry import (
+    destination_registry,
+    source_registry,
+)
+
+src_cmds = [SrcCmd(cmd_name=k, registry_entry=v) for k, v in source_registry.items()]
+dest_cmds = [DestCmd(cmd_name=k, registry_entry=v) for k, v in destination_registry.items()]
+
+src: list[click.Group] = [v.get_cmd() for v in src_cmds]
+
+dest: list[click.Command] = [v.get_cmd() for v in dest_cmds]