unstructured-ingest 0.0.3__tar.gz → 0.0.5__tar.gz

unstructured-ingest-0.0.5/LICENSE.md

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2022 Unstructured Technologies, Inc
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

{unstructured-ingest-0.0.3 → unstructured-ingest-0.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.0.3
+Version: 0.0.5
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -21,6 +21,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
+Provides-Extra: remote
 Provides-Extra: csv
 Provides-Extra: doc
 Provides-Extra: docx
@@ -37,7 +38,7 @@ Provides-Extra: rst
 Provides-Extra: tsv
 Provides-Extra: xlsx
 Provides-Extra: airtable
-Provides-Extra: astra
+Provides-Extra: astradb
 Provides-Extra: azure
 Provides-Extra: azure-cognitive-search
 Provides-Extra: biomed
@@ -45,6 +46,7 @@ Provides-Extra: box
 Provides-Extra: chroma
 Provides-Extra: clarifai
 Provides-Extra: confluence
+Provides-Extra: couchbase
 Provides-Extra: delta-table
 Provides-Extra: discord
 Provides-Extra: dropbox
@@ -56,6 +58,7 @@ Provides-Extra: google-drive
 Provides-Extra: hubspot
 Provides-Extra: jira
 Provides-Extra: kafka
+Provides-Extra: kdbai
 Provides-Extra: milvus
 Provides-Extra: mongodb
 Provides-Extra: notion
@@ -75,12 +78,14 @@ Provides-Extra: wikipedia
 Provides-Extra: weaviate
 Provides-Extra: databricks-volumes
 Provides-Extra: singlestore
+Provides-Extra: vectara
 Provides-Extra: embed-huggingface
 Provides-Extra: embed-octoai
 Provides-Extra: embed-vertexai
 Provides-Extra: embed-voyageai
 Provides-Extra: openai
 Provides-Extra: bedrock
+License-File: LICENSE.md
 
 # Unstructured Ingest  
 

{unstructured-ingest-0.0.3 → unstructured-ingest-0.0.5}/setup.py

@@ -81,7 +81,7 @@ all_doc_reqs = list(
 )
 connectors_reqs = {
     "airtable": load_requirements("requirements/connectors/airtable.in"),
-    "astra": load_requirements("requirements/connectors/astra.in"),
+    "astradb": load_requirements("requirements/connectors/astradb.in"),
     "azure": load_requirements("requirements/connectors/azure.in"),
     "azure-cognitive-search": load_requirements(
         "requirements/connectors/azure-cognitive-search.in",
@@ -91,6 +91,7 @@ connectors_reqs = {
     "chroma": load_requirements("requirements/connectors/chroma.in"),
     "clarifai": load_requirements("requirements/connectors/clarifai.in"),
     "confluence": load_requirements("requirements/connectors/confluence.in"),
+    "couchbase": load_requirements("requirements/connectors/couchbase.in"),
     "delta-table": load_requirements("requirements/connectors/delta-table.in"),
     "discord": load_requirements("requirements/connectors/discord.in"),
     "dropbox": load_requirements("requirements/connectors/dropbox.in"),
@@ -102,6 +103,7 @@ connectors_reqs = {
     "hubspot": load_requirements("requirements/connectors/hubspot.in"),
     "jira": load_requirements("requirements/connectors/jira.in"),
     "kafka": load_requirements("requirements/connectors/kafka.in"),
+    "kdbai": load_requirements("requirements/connectors/kdbai.in"),
     "milvus": load_requirements("requirements/connectors/milvus.in"),
     "mongodb": load_requirements("requirements/connectors/mongodb.in"),
     "notion": load_requirements("requirements/connectors/notion.in"),
@@ -121,6 +123,7 @@ connectors_reqs = {
     "weaviate": load_requirements("requirements/connectors/weaviate.in"),
     "databricks-volumes": load_requirements("requirements/connectors/databricks-volumes.in"),
     "singlestore": load_requirements("requirements/connectors/singlestore.in"),
+    "vectara": load_requirements("requirements/connectors/vectara.in"),
 }
 
 embed_reqs = {
@@ -150,7 +153,9 @@ docs_reqs = {
     "xlsx": xlsx_reqs,
 }
 
-extras_require = {}
+extras_require = {
+    "remote": load_requirements("requirements/remote/client.in"),
+}
 for d in [docs_reqs, connectors_reqs, embed_reqs]:
     extras_require.update(d)
 

unstructured-ingest-0.0.5/test/test_chunking_utils.py

@@ -0,0 +1,36 @@
+from pathlib import Path
+
+import pytest
+from unstructured.chunking import dispatch
+from unstructured.documents.elements import assign_and_map_hash_ids
+from unstructured.partition.auto import partition
+
+from unstructured_ingest.utils.chunking import (
+    assign_and_map_hash_ids as new_assign_and_map_hash_ids,
+)
+
+test_file_path = Path(__file__).resolve()
+project_root = test_file_path.parent.parent
+docs_path = project_root / "example-docs"
+
+
+@pytest.mark.parametrize(
+    "chunking_strategy",
+    ["basic", "by_title"],
+)
+def test_assign_and_map_hash_ids(chunking_strategy):
+    # Make sure the new logic working on dict content matches the
+    # results if using the unstructured version
+    file_path = docs_path / "book-war-and-peace-1p.txt"
+    elements = partition(filename=str(file_path.resolve()), strategy="fast")
+    chunked_elements = dispatch.chunk(elements=elements, chunking_strategy=chunking_strategy)
+    chunked_elements_copy = chunked_elements.copy()
+
+    hashed_chunked_elements = assign_and_map_hash_ids(chunked_elements)
+    og_chunked_elements_dicts = [e.to_dict() for e in hashed_chunked_elements]
+
+    new_chunked_elements_dicts = [e.to_dict() for e in chunked_elements_copy]
+    new_chunked_elements_dicts = new_assign_and_map_hash_ids(new_chunked_elements_dicts)
+
+    for e1, e2 in zip(og_chunked_elements_dicts, new_chunked_elements_dicts):
+        assert e1 == e2

{unstructured-ingest-0.0.3 → unstructured-ingest-0.0.5}/test/test_interfaces.py

@@ -90,7 +90,7 @@ class ExampleIngestDoc(BaseSingleIngestDoc):
         pass
 
 
-@pytest.fixture()
+@pytest.fixture
 def partition_test_results():
     # Reusable partition test results, calculated only once
     result = partition(
@@ -107,7 +107,7 @@ def partition_test_results():
     return result
 
 
-@pytest.fixture()
+@pytest.fixture
 def partition_file_test_results(partition_test_results):
     # Reusable partition_file test results, calculated only once
     return elements_to_dicts(partition_test_results)

unstructured-ingest-0.0.5/test/test_utils_v2.py

@@ -0,0 +1,82 @@
+import json
+from typing import Any
+
+from pydantic import BaseModel, Field, Secret, SecretStr
+from pydantic.types import _SecretBase
+
+from unstructured_ingest.v2.utils import serialize_base_model, serialize_base_model_json
+
+
+class MockChildBaseModel(BaseModel):
+    child_secret_str: SecretStr
+    child_secret_float: Secret[float]
+    child_not_secret_dict: dict[str, Any] = Field(default_factory=dict)
+
+
+class MockBaseModel(BaseModel):
+    secret_str: SecretStr
+    not_secret_bool: bool
+    secret_child_base: Secret[MockChildBaseModel]
+    not_secret_list: list[int] = Field(default_factory=list)
+
+
+model = MockBaseModel(
+    secret_str="secret string",
+    not_secret_bool=False,
+    secret_child_base=MockChildBaseModel(
+        child_secret_str="child secret string",
+        child_secret_float=3.14,
+        child_not_secret_dict={"key": "value"},
+    ),
+    not_secret_list=[1, 2, 3],
+)
+
+
+def test_serialize_base_model():
+
+    serialized_dict = model.dict()
+    assert isinstance(serialized_dict["secret_str"], _SecretBase)
+    assert isinstance(serialized_dict["secret_child_base"], _SecretBase)
+
+    serialized_dict_w_secrets = serialize_base_model(model=model)
+    assert not isinstance(serialized_dict_w_secrets["secret_str"], _SecretBase)
+    assert not isinstance(serialized_dict_w_secrets["secret_child_base"], _SecretBase)
+
+    expected_dict = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+
+    assert serialized_dict_w_secrets == expected_dict
+
+
+def test_serialize_base_model_json():
+    serialized_json = model.json()
+    serialized_dict = json.loads(serialized_json)
+    expected_dict = {
+        "secret_str": "**********",
+        "not_secret_bool": False,
+        "secret_child_base": "**********",
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict == serialized_dict
+
+    serialized_json_w_secrets = serialize_base_model_json(model=model)
+    serialized_dict_w_secrets = json.loads(serialized_json_w_secrets)
+    expected_dict_w_secrets = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict_w_secrets == serialized_dict_w_secrets

unstructured-ingest-0.0.5/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.0.5"  # pragma: no cover

{unstructured-ingest-0.0.3 → unstructured-ingest-0.0.5}/unstructured_ingest/cli/cli.py

@@ -1,16 +1,21 @@
+from typing import TYPE_CHECKING
+
 import click
 
 from unstructured_ingest.cli import dest, src
 from unstructured_ingest.v2.cli.cmds import dest as dest_v2
 from unstructured_ingest.v2.cli.cmds import src as src_v2
 
+if TYPE_CHECKING:
+    from click import Command
+
 
 @click.group()
 def ingest():
     pass
 
 
-def get_cmd() -> click.Command:
+def get_cmd() -> "Command":
     """Construct and return a Click command object representing the main command for the CLI.
 
     This function adds all dest_subcommand(s) to each src_subcommand, and adds all of those

{unstructured-ingest-0.0.3 → unstructured-ingest-0.0.5}/unstructured_ingest/cli/cmds/__init__.py

@@ -7,8 +7,8 @@ from unstructured_ingest.cli.base.src import BaseSrcCmd
 from unstructured_ingest.cli.cmds.fsspec.sftp import get_base_src_cmd as sftp_base_src_cmd
 
 from .airtable import get_base_src_cmd as airtable_base_src_cmd
-from .astra import get_base_dest_cmd as astra_base_dest_cmd
-from .astra import get_base_src_cmd as astra_base_src_cmd
+from .astradb import get_base_dest_cmd as astradb_base_dest_cmd
+from .astradb import get_base_src_cmd as astradb_base_src_cmd
 from .azure_cognitive_search import get_base_dest_cmd as azure_cognitive_search_base_dest_cmd
 from .biomed import get_base_src_cmd as biomed_base_src_cmd
 from .chroma import get_base_dest_cmd as chroma_base_dest_cmd
@@ -63,7 +63,7 @@ if t.TYPE_CHECKING:
 
 base_src_cmd_fns: t.List[t.Callable[[], BaseSrcCmd]] = [
     airtable_base_src_cmd,
-    astra_base_src_cmd,
+    astradb_base_src_cmd,
     azure_base_src_cmd,
     biomed_base_src_cmd,
     box_base_src_cmd,
@@ -106,7 +106,7 @@ if src_duplicates:
     )
 
 base_dest_cmd_fns: t.List[t.Callable[[], "BaseDestCmd"]] = [
-    astra_base_dest_cmd,
+    astradb_base_dest_cmd,
     azure_base_dest_cmd,
     box_base_dest_cmd,
     chroma_base_dest_cmd,

unstructured-ingest-0.0.3/unstructured_ingest/cli/cmds/astra.py → unstructured-ingest-0.0.5/unstructured_ingest/cli/cmds/astradb.py

@@ -4,11 +4,11 @@ from dataclasses import dataclass
 import click
 
 from unstructured_ingest.cli.interfaces import CliConfig, Dict
-from unstructured_ingest.connector.astra import AstraWriteConfig, SimpleAstraConfig
+from unstructured_ingest.connector.astradb import AstraDBWriteConfig, SimpleAstraDBConfig
 
 
 @dataclass
-class AstraCliConfig(SimpleAstraConfig, CliConfig):
+class AstraDBCliConfig(SimpleAstraDBConfig, CliConfig):
     @staticmethod
     def get_cli_options() -> t.List[click.Option]:
         options = [
@@ -48,7 +48,7 @@ class AstraCliConfig(SimpleAstraConfig, CliConfig):
 
 
 @dataclass
-class AstraCliWriteConfig(AstraWriteConfig, CliConfig):
+class AstraDBCliWriteConfig(AstraDBWriteConfig, CliConfig):
     @staticmethod
     def get_cli_options() -> t.List[click.Option]:
         options = [
@@ -81,8 +81,8 @@ def get_base_src_cmd():
     from unstructured_ingest.cli.base.src import BaseSrcCmd
 
     cmd_cls = BaseSrcCmd(
-        cmd_name="astra",
-        cli_config=AstraCliConfig,
+        cmd_name="astradb",
+        cli_config=AstraDBCliConfig,
     )
     return cmd_cls
 
@@ -91,9 +91,9 @@ def get_base_dest_cmd():
     from unstructured_ingest.cli.base.dest import BaseDestCmd
 
     cmd_cls = BaseDestCmd(
-        cmd_name="astra",
-        cli_config=AstraCliConfig,
-        additional_cli_options=[AstraCliWriteConfig],
-        write_config=AstraWriteConfig,
+        cmd_name="astradb",
+        cli_config=AstraDBCliConfig,
+        additional_cli_options=[AstraDBCliWriteConfig],
+        write_config=AstraDBWriteConfig,
     )
     return cmd_cls

{unstructured-ingest-0.0.3 → unstructured-ingest-0.0.5}/unstructured_ingest/cli/interfaces.py

@@ -11,7 +11,6 @@ from pathlib import Path
 import click
 from dataclasses_json.core import Json
 from typing_extensions import Self
-from unstructured.chunking import CHUNK_MAX_CHARS_DEFAULT, CHUNK_MULTI_PAGE_DEFAULT
 
 from unstructured_ingest.interfaces import (
     BaseConfig,
@@ -25,6 +24,9 @@ from unstructured_ingest.interfaces import (
     RetryStrategyConfig,
 )
 
+CHUNK_MAX_CHARS_DEFAULT: int = 500
+CHUNK_MULTI_PAGE_DEFAULT: bool = True
+
 
 class Dict(click.ParamType):
     name = "dict"
@@ -412,14 +414,19 @@ class CliFilesStorageConfig(FileStorageConfig, CliMixin):
 class CliEmbeddingConfig(EmbeddingConfig, CliMixin):
     @staticmethod
     def get_cli_options() -> t.List[click.Option]:
-        from unstructured.embed import EMBEDDING_PROVIDER_TO_CLASS_MAP
-
+        embed_providers = [
+            "langchain-openai",
+            "langchain-huggingface",
+            "langchain-aws-bedrock",
+            "langchain-vertexai",
+            "langchain-voyageai",
+            "octoai",
+        ]
         options = [
             click.Option(
                 ["--embedding-provider"],
-                help="Type of the embedding class to be used. Can be one of: "
-                f"{list(EMBEDDING_PROVIDER_TO_CLASS_MAP)}",
-                type=click.Choice(list(EMBEDDING_PROVIDER_TO_CLASS_MAP)),
+                help="Type of the embedding class to be used.",
+                type=click.Choice(embed_providers),
             ),
             click.Option(
                 ["--embedding-api-key"],