unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/utils/table.py

@@ -0,0 +1,73 @@
+from typing import Any
+
+import pandas as pd
+
+from unstructured_ingest.utils.data_prep import flatten_dict
+
+
+def get_default_pandas_dtypes() -> dict[str, Any]:
+    return {
+        "text": pd.StringDtype(),  # type: ignore
+        "type": pd.StringDtype(),  # type: ignore
+        "element_id": pd.StringDtype(),  # type: ignore
+        "filename": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "filetype": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "file_directory": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "last_modified": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "attached_to_filename": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "parent_id": pd.StringDtype(),  # Optional[str],  # type: ignore
+        "category_depth": "Int64",  # Optional[int]
+        "image_path": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "languages": object,  # Optional[list[str]]
+        "page_number": "Int64",  # Optional[int]
+        "page_name": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "url": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "link_urls": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "link_texts": object,  # Optional[list[str]]
+        "links": object,
+        "sent_from": object,  # Optional[list[str]],
+        "sent_to": object,  # Optional[list[str]]
+        "subject": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "section": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "header_footer_type": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "emphasized_text_contents": object,  # Optional[list[str]]
+        "emphasized_text_tags": object,  # Optional[list[str]]
+        "text_as_html": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "regex_metadata": object,
+        "max_characters": "Int64",  # Optional[int]
+        "is_continuation": "boolean",  # Optional[bool]
+        "detection_class_prob": float,  # Optional[float],
+        "sender": pd.StringDtype(),  # type: ignore
+        "coordinates_points": object,
+        "coordinates_system": pd.StringDtype(),  # type: ignore
+        "coordinates_layout_width": float,
+        "coordinates_layout_height": float,
+        "data_source_url": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_version": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_record_locator": object,
+        "data_source_date_created": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_date_modified": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_date_processed": pd.StringDtype(),  # Optional[str]  # type: ignore
+        "data_source_permissions_data": object,
+        "embeddings": object,
+        "regex_metadata_key": object,
+    }
+
+
+def convert_to_pandas_dataframe(
+    elements_dict: list[dict[str, Any]],
+    drop_empty_cols: bool = False,
+) -> pd.DataFrame:
+    # Flatten metadata if it hasn't already been flattened
+    for d in elements_dict:
+        if metadata := d.pop("metadata", None):
+            d.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
+
+    df = pd.DataFrame.from_dict(
+        elements_dict,
+    )
+    dt = {k: v for k, v in get_default_pandas_dtypes().items() if k in df.columns}
+    df = df.astype(dt)
+    if drop_empty_cols:
+        df.dropna(axis=1, how="all", inplace=True)
+    return df

unstructured_ingest/v2/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

unstructured_ingest/v2/cli/base/__init__.py

@@ -0,0 +1,4 @@
+from .dest import DestCmd
+from .src import SrcCmd
+
+__all__ = ["SrcCmd", "DestCmd"]

unstructured_ingest/v2/cli/base/cmd.py

@@ -0,0 +1,215 @@
+import inspect
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field, fields
+from typing import Any, Optional, Type, TypeVar
+
+import click
+
+from unstructured_ingest.v2.cli.base.importer import import_from_string
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.cli.utils import extract_config
+from unstructured_ingest.v2.interfaces import ProcessorConfig
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.pipeline.pipeline import Pipeline
+from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
+from unstructured_ingest.v2.processes.connector_registry import (
+    DownloaderT,
+    IndexerT,
+    UploaderT,
+    UploadStager,
+    UploadStagerConfig,
+    UploadStagerT,
+    destination_registry,
+    source_registry,
+)
+from unstructured_ingest.v2.processes.connectors.local import LocalUploader, LocalUploaderConfig
+from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
+
+CommandT = TypeVar("CommandT", bound=click.Command)
+
+
+@dataclass
+class BaseCmd(ABC):
+    cmd_name: str
+    default_configs: list[Type[CliConfig]] = field(default_factory=list)
+
+    @property
+    def cmd_name_key(self):
+        return self.cmd_name.replace("-", "_")
+
+    @property
+    def cli_cmd_name(self):
+        return self.cmd_name.replace("_", "-")
+
+    @abstractmethod
+    def cmd(self, ctx: click.Context, **options) -> None:
+        pass
+
+    def add_options(self, cmd: CommandT, extras: list[Type[CliConfig]]) -> CommandT:
+        configs = self.default_configs
+        # make sure what's unique to this cmd appears first
+        extras.extend(configs)
+        for config in extras:
+            try:
+                config.add_cli_options(cmd=cmd)
+            except ValueError as e:
+                raise ValueError(f"failed to set configs from {config.__name__}: {e}")
+        return cmd
+
+    def get_pipline(
+        self,
+        src: str,
+        source_options: dict[str, Any],
+        dest: Optional[str] = None,
+        destination_options: Optional[dict[str, Any]] = None,
+    ) -> Pipeline:
+        logger.debug(
+            f"creating pipeline from cli using source {src} with options: {source_options}"
+        )
+        pipeline_kwargs: dict[str, Any] = {
+            "context": self.get_processor_config(options=source_options),
+            "downloader": self.get_downloader(src=src, options=source_options),
+            "indexer": self.get_indexer(src=src, options=source_options),
+            "partitioner": self.get_partitioner(options=source_options),
+        }
+        if chunker := self.get_chunker(options=source_options):
+            pipeline_kwargs["chunker"] = chunker
+        if embedder := self.get_embeder(options=source_options):
+            pipeline_kwargs["embedder"] = embedder
+        if dest:
+            logger.debug(
+                f"setting destination on pipeline {dest} with options: {destination_options}"
+            )
+            if uploader_stager := self.get_upload_stager(dest=dest, options=destination_options):
+                pipeline_kwargs["stager"] = uploader_stager
+            pipeline_kwargs["uploader"] = self.get_uploader(dest=dest, options=destination_options)
+        else:
+            # Default to local uploader
+            # TODO remove after v1 no longer supported
+            destination_options = destination_options or {}
+            if "output_dir" not in destination_options:
+                destination_options["output_dir"] = source_options["output_dir"]
+            pipeline_kwargs["uploader"] = self.get_default_uploader(options=destination_options)
+        return Pipeline(**pipeline_kwargs)
+
+    @staticmethod
+    def get_default_uploader(options: dict[str, Any]) -> UploaderT:
+        uploader_config = extract_config(flat_data=options, config=LocalUploaderConfig)
+        return LocalUploader(upload_config=uploader_config)
+
+    @staticmethod
+    def get_chunker(options: dict[str, Any]) -> Optional[Chunker]:
+        chunker_config = extract_config(flat_data=options, config=ChunkerConfig)
+        if not chunker_config.chunking_strategy:
+            return None
+        return Chunker(config=chunker_config)
+
+    @staticmethod
+    def get_embeder(options: dict[str, Any]) -> Optional[Embedder]:
+        embedder_config = extract_config(flat_data=options, config=EmbedderConfig)
+        if not embedder_config.embedding_provider:
+            return None
+        return Embedder(config=embedder_config)
+
+    @staticmethod
+    def get_partitioner(options: dict[str, Any]) -> Partitioner:
+        partitioner_config = extract_config(flat_data=options, config=PartitionerConfig)
+        return Partitioner(config=partitioner_config)
+
+    @staticmethod
+    def get_processor_config(options: dict[str, Any]) -> ProcessorConfig:
+        return extract_config(flat_data=options, config=ProcessorConfig)
+
+    @staticmethod
+    def get_indexer(src: str, options: dict[str, Any]) -> IndexerT:
+        source_entry = source_registry[src]
+        indexer_kwargs: dict[str, Any] = {}
+        if indexer_config_cls := source_entry.indexer_config:
+            indexer_kwargs["index_config"] = extract_config(
+                flat_data=options, config=indexer_config_cls
+            )
+        if connection_config_cls := source_entry.connection_config:
+            indexer_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        indexer_cls = source_entry.indexer
+        return indexer_cls(**indexer_kwargs)
+
+    @staticmethod
+    def get_downloader(src: str, options: dict[str, Any]) -> DownloaderT:
+        source_entry = source_registry[src]
+        downloader_kwargs: dict[str, Any] = {}
+        if downloader_config_cls := source_entry.downloader_config:
+            downloader_kwargs["download_config"] = extract_config(
+                flat_data=options, config=downloader_config_cls
+            )
+        if connection_config_cls := source_entry.connection_config:
+            downloader_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        downloader_cls = source_entry.downloader
+        return downloader_cls(**downloader_kwargs)
+
+    @staticmethod
+    def get_custom_stager(
+        stager_reference: str, stager_config_kwargs: Optional[dict] = None
+    ) -> Optional[UploadStagerT]:
+        uploader_cls = import_from_string(stager_reference)
+        if not inspect.isclass(uploader_cls):
+            raise ValueError(
+                f"custom stager must be a reference to a python class, got: {type(uploader_cls)}"
+            )
+        if not issubclass(uploader_cls, UploadStager):
+            raise ValueError(
+                "custom stager must be an implementation of the UploadStager interface"
+            )
+        fields_dict = {f.name: f.type for f in fields(uploader_cls)}
+        upload_stager_config_cls = fields_dict["upload_stager_config"]
+        if not inspect.isclass(upload_stager_config_cls):
+            raise ValueError(
+                f"custom stager config must be a class, got: {type(upload_stager_config_cls)}"
+            )
+        if not issubclass(upload_stager_config_cls, UploadStagerConfig):
+            raise ValueError(
+                "custom stager config must be an implementation "
+                "of the UploadStagerUploadStagerConfig interface"
+            )
+        upload_stager_kwargs: dict[str, Any] = {}
+        if stager_config_kwargs:
+            upload_stager_kwargs["upload_stager_config"] = upload_stager_config_cls(
+                **stager_config_kwargs
+            )
+        return uploader_cls(**upload_stager_kwargs)
+
+    @staticmethod
+    def get_upload_stager(dest: str, options: dict[str, Any]) -> Optional[UploadStagerT]:
+        if custom_stager := options.get("custom_stager"):
+            return BaseCmd.get_custom_stager(
+                stager_reference=custom_stager,
+                stager_config_kwargs=options.get("custom_stager_config_kwargs"),
+            )
+        dest_entry = destination_registry[dest]
+        upload_stager_kwargs: dict[str, Any] = {}
+        if upload_stager_config_cls := dest_entry.upload_stager_config:
+            upload_stager_kwargs["upload_stager_config"] = extract_config(
+                flat_data=options, config=upload_stager_config_cls
+            )
+        if upload_stager_cls := dest_entry.upload_stager:
+            return upload_stager_cls(**upload_stager_kwargs)
+        return None
+
+    @staticmethod
+    def get_uploader(dest, options: dict[str, Any]) -> UploaderT:
+        dest_entry = destination_registry[dest]
+        uploader_kwargs: dict[str, Any] = {}
+        if uploader_config_cls := dest_entry.uploader_config:
+            uploader_kwargs["upload_config"] = extract_config(
+                flat_data=options, config=uploader_config_cls
+            )
+        if connection_config_cls := dest_entry.connection_config:
+            uploader_kwargs["connection_config"] = extract_config(
+                flat_data=options, config=connection_config_cls
+            )
+        uploader_cls = dest_entry.uploader
+        return uploader_cls(**uploader_kwargs)

unstructured_ingest/v2/cli/base/dest.py

@@ -0,0 +1,76 @@
+import logging
+from dataclasses import dataclass
+from typing import Optional, Type
+
+import click
+
+from unstructured_ingest.v2.cli.base.cmd import BaseCmd
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.cli.utils import Dict, conform_click_options
+from unstructured_ingest.v2.logger import logger
+
+
+@dataclass
+class DestCmd(BaseCmd):
+    connection_config: Optional[Type[CliConfig]] = None
+    uploader_config: Optional[Type[CliConfig]] = None
+    upload_stager_config: Optional[Type[CliConfig]] = None
+
+    def cmd(self, ctx: click.Context, **options) -> None:
+        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
+        if not ctx.parent:
+            raise click.ClickException("destination command called without a parent")
+        if not ctx.parent.info_name:
+            raise click.ClickException("parent command missing info name")
+        source_cmd = ctx.parent.info_name.replace("-", "_")
+        source_options: dict = ctx.parent.params if ctx.parent else {}
+        conform_click_options(options)
+        try:
+            pipeline = self.get_pipline(
+                src=source_cmd,
+                source_options=source_options,
+                dest=self.cmd_name,
+                destination_options=options,
+            )
+            pipeline.run()
+        except Exception as e:
+            logger.error(f"failed to run destination command {self.cmd_name}: {e}", exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_cmd(self) -> click.Command:
+        # Dynamically create the command without the use of click decorators
+        fn = self.cmd
+        fn = click.pass_context(fn)
+        cmd = click.command(fn)
+        if not isinstance(cmd, click.core.Command):
+            raise ValueError(f"generated command was not of expected type Command: {type(cmd)}")
+        cmd.name = self.cli_cmd_name
+        cmd.short_help = "v2"
+        cmd.invoke_without_command = True
+        extras = [
+            x
+            for x in [self.uploader_config, self.upload_stager_config, self.connection_config]
+            if x
+        ]
+        self.add_options(cmd, extras=extras)
+        cmd.params.append(
+            click.Option(
+                ["--custom-stager"],
+                required=False,
+                type=str,
+                default=None,
+                help="Pass a pointer to a custom upload stager to use, "
+                "must be in format '<module>:<attribute>'",
+            )
+        )
+        cmd.params.append(
+            click.Option(
+                ["--custom-stager-config-kwargs"],
+                required=False,
+                type=Dict(),
+                default=None,
+                help="Any kwargs to instantiate the configuration "
+                "associated with the customer stager",
+            )
+        )
+        return cmd

unstructured_ingest/v2/cli/base/importer.py

@@ -0,0 +1,34 @@
+import importlib
+from typing import Any
+
+
+class ImportFromStringError(Exception):
+    pass
+
+
+def import_from_string(import_str: Any) -> Any:
+    if not isinstance(import_str, str):
+        return import_str
+
+    module_str, _, attrs_str = import_str.partition(":")
+    if not module_str or not attrs_str:
+        message = 'Import string "{import_str}" must be in format "<module>:<attribute>".'
+        raise ImportFromStringError(message.format(import_str=import_str))
+
+    try:
+        module = importlib.import_module(module_str)
+    except ModuleNotFoundError as exc:
+        if exc.name != module_str:
+            raise exc from None
+        message = 'Could not import module "{module_str}".'
+        raise ImportFromStringError(message.format(module_str=module_str))
+
+    instance = module
+    try:
+        for attr_str in attrs_str.split("."):
+            instance = getattr(instance, attr_str)
+    except AttributeError:
+        message = 'Attribute "{attrs_str}" not found in module "{module_str}".'
+        raise ImportFromStringError(message.format(attrs_str=attrs_str, module_str=module_str))
+
+    return instance

unstructured_ingest/v2/cli/base/src.py

@@ -0,0 +1,70 @@
+import logging
+from dataclasses import dataclass, field
+from typing import Optional, Type
+
+import click
+
+from unstructured_ingest.v2.cli.base.cmd import BaseCmd
+from unstructured_ingest.v2.cli.configs import (
+    ChunkerCliConfig,
+    EmbedderCliConfig,
+    PartitionerCliConfig,
+    ProcessorCliConfig,
+)
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.cli.utils import Group, conform_click_options
+from unstructured_ingest.v2.logger import logger
+
+
+@dataclass
+class SrcCmd(BaseCmd):
+    indexer_config: Optional[Type[CliConfig]] = None
+    downloader_config: Optional[Type[CliConfig]] = None
+    connection_config: Optional[Type[CliConfig]] = None
+    default_configs: list[CliConfig] = field(
+        default_factory=lambda: [
+            ProcessorCliConfig,
+            PartitionerCliConfig,
+            EmbedderCliConfig,
+            ChunkerCliConfig,
+        ]
+    )
+
+    def cmd(self, ctx: click.Context, **options) -> None:
+        if ctx.invoked_subcommand:
+            return
+
+        conform_click_options(options)
+        logger.setLevel(logging.DEBUG if options.get("verbose", False) else logging.INFO)
+        try:
+            pipeline = self.get_pipline(src=self.cmd_name, source_options=options)
+            pipeline.run()
+        except Exception as e:
+            logger.error(f"failed to run source command {self.cmd_name}: {e}", exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_cmd(self) -> click.Group:
+        # Dynamically create the command without the use of click decorators
+        fn = self.cmd
+        fn = click.pass_context(fn)
+        cmd = click.group(fn, cls=Group)
+        if not isinstance(cmd, click.core.Group):
+            raise ValueError(f"generated src command was not of expected type Group: {type(cmd)}")
+        cmd.name = self.cli_cmd_name
+        cmd.short_help = "v2"
+        cmd.invoke_without_command = True
+        extras = [
+            x for x in [self.indexer_config, self.downloader_config, self.connection_config] if x
+        ]
+        self.add_options(cmd, extras=extras)
+
+        # TODO remove after v1 no longer supported
+        cmd.params.append(
+            click.Option(
+                ["--output-dir"],
+                required=False,
+                type=str,
+                help="Local path to write partitioned output to",
+            )
+        )
+        return cmd

unstructured_ingest/v2/cli/cli.py

@@ -0,0 +1,24 @@
+import click
+
+from unstructured_ingest.v2.cli.cmds import dest, src
+
+
+@click.group()
+def ingest():
+    pass
+
+
+def get_cmd() -> click.Command:
+    """Construct and return a Click command object representing the main command for the CLI.
+
+    This function adds all dest_subcommand(s) to each src_subcommand, and adds all of those
+    to the main command as nested subcommands.
+    """
+    cmd = ingest
+    # Add all subcommands
+    for src_subcommand in src:
+        # Add all destination subcommands
+        for dest_subcommand in dest:
+            src_subcommand.add_command(dest_subcommand)
+        cmd.add_command(src_subcommand)
+    return cmd

unstructured_ingest/v2/cli/cmds/__init__.py

@@ -0,0 +1,87 @@
+from collections import Counter
+
+import click
+
+from .astra import astra_dest_cmd
+from .azure_cognitive_search import azure_cognitive_search_dest_cmd
+from .chroma import chroma_dest_cmd
+from .databricks_volumes import databricks_volumes_dest_cmd
+from .elasticsearch import elasticsearch_dest_cmd, elasticsearch_src_cmd
+from .fsspec.azure import azure_dest_cmd, azure_src_cmd
+from .fsspec.box import box_dest_cmd, box_src_cmd
+from .fsspec.dropbox import dropbox_dest_cmd, dropbox_src_cmd
+from .fsspec.gcs import gcs_dest_cmd, gcs_src_cmd
+from .fsspec.s3 import s3_dest_cmd, s3_src_cmd
+from .fsspec.sftp import sftp_dest_cmd, sftp_src_cmd
+from .google_drive import google_drive_src_cmd
+from .local import local_dest_cmd, local_src_cmd
+from .mongodb import mongodb_dest_cmd
+from .onedrive import onedrive_drive_src_cmd
+from .opensearch import opensearch_dest_cmd, opensearch_src_cmd
+from .pinecone import pinecone_dest_cmd
+from .salesforce import salesforce_src_cmd
+from .sharepoint import sharepoint_drive_src_cmd
+from .singlestore import singlestore_dest_cmd
+from .sql import sql_dest_cmd
+from .weaviate import weaviate_dest_cmd
+
+src_cmds = [
+    azure_src_cmd,
+    box_src_cmd,
+    dropbox_src_cmd,
+    elasticsearch_src_cmd,
+    gcs_src_cmd,
+    google_drive_src_cmd,
+    local_src_cmd,
+    onedrive_drive_src_cmd,
+    opensearch_src_cmd,
+    s3_src_cmd,
+    salesforce_src_cmd,
+    sharepoint_drive_src_cmd,
+    sftp_src_cmd,
+]
+duplicate_src_names = [
+    name for name, count in Counter([s.cmd_name for s in src_cmds]).items() if count > 1
+]
+if duplicate_src_names:
+    raise ValueError(
+        "the following source cmd names were reused, all must be unique: {}".format(
+            ", ".join(duplicate_src_names)
+        )
+    )
+
+dest_cmds = [
+    astra_dest_cmd,
+    azure_cognitive_search_dest_cmd,
+    azure_dest_cmd,
+    box_dest_cmd,
+    chroma_dest_cmd,
+    dropbox_dest_cmd,
+    elasticsearch_dest_cmd,
+    gcs_dest_cmd,
+    local_dest_cmd,
+    opensearch_dest_cmd,
+    pinecone_dest_cmd,
+    s3_dest_cmd,
+    sftp_dest_cmd,
+    singlestore_dest_cmd,
+    weaviate_dest_cmd,
+    mongodb_dest_cmd,
+    databricks_volumes_dest_cmd,
+    sql_dest_cmd,
+]
+
+duplicate_dest_names = [
+    name for name, count in Counter([d.cmd_name for d in dest_cmds]).items() if count > 1
+]
+if duplicate_dest_names:
+    raise ValueError(
+        "the following dest cmd names were reused, all must be unique: {}".format(
+            ", ".join(duplicate_dest_names)
+        )
+    )
+
+
+src: list[click.Group] = [v.get_cmd() for v in src_cmds]
+
+dest: list[click.Command] = [v.get_cmd() for v in dest_cmds]

unstructured_ingest/v2/cli/cmds/astra.py

@@ -0,0 +1,85 @@
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.v2.cli.base import DestCmd
+from unstructured_ingest.v2.cli.interfaces import CliConfig
+from unstructured_ingest.v2.cli.utils import Dict
+from unstructured_ingest.v2.processes.connectors.astra import CONNECTOR_TYPE
+
+
+@dataclass
+class AstraCliConnectionConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--token"],
+                required=True,
+                type=str,
+                help="Astra DB Token with access to the database.",
+                envvar="ASTRA_DB_TOKEN",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--api-endpoint"],
+                required=True,
+                type=str,
+                help="The API endpoint for the Astra DB.",
+                envvar="ASTRA_DB_ENDPOINT",
+                show_envvar=True,
+            ),
+        ]
+        return options
+
+
+@dataclass
+class AstraCliUploaderConfig(CliConfig):
+    @staticmethod
+    def get_cli_options() -> list[click.Option]:
+        options = [
+            click.Option(
+                ["--collection-name"],
+                required=False,
+                type=str,
+                help="The name of the Astra DB collection. "
+                "Note that the collection name must only include letters, "
+                "numbers, and underscores.",
+            ),
+            click.Option(
+                ["--embedding-dimension"],
+                required=True,
+                default=384,
+                type=int,
+                help="The dimensionality of the embeddings",
+            ),
+            click.Option(
+                ["--namespace"],
+                required=False,
+                default=None,
+                type=str,
+                help="The Astra DB connection namespace.",
+            ),
+            click.Option(
+                ["--requested-indexing-policy"],
+                required=False,
+                default=None,
+                type=Dict(),
+                help="The indexing policy to use for the collection."
+                'example: \'{"deny": ["metadata"]}\' ',
+            ),
+            click.Option(
+                ["--batch-size"],
+                default=20,
+                type=int,
+                help="Number of records per batch",
+            ),
+        ]
+        return options
+
+
+astra_dest_cmd = DestCmd(
+    cmd_name=CONNECTOR_TYPE,
+    connection_config=AstraCliConnectionConfig,
+    uploader_config=AstraCliUploaderConfig,
+)