unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/cli/cli.py

@@ -0,0 +1,37 @@
+from typing import TYPE_CHECKING
+
+import click
+
+from unstructured_ingest.cli import dest, src
+from unstructured_ingest.v2.cli.cmds import dest as dest_v2
+from unstructured_ingest.v2.cli.cmds import src as src_v2
+
+if TYPE_CHECKING:
+    from click import Command
+
+
+@click.group()
+def ingest():
+    pass
+
+
+def get_cmd() -> "Command":
+    """Construct and return a Click command object representing the main command for the CLI.
+
+    This function adds all dest_subcommand(s) to each src_subcommand, and adds all of those
+    to the main command as nested subcommands.
+    """
+    cmd = ingest
+    src_dict = {s.name: s for s in src}
+    dest_dict = {d.name: d for d in dest}
+    for s in src_v2:
+        src_dict[s.name] = s
+    for d in dest_v2:
+        dest_dict[d.name] = d
+    # Add all subcommands
+    for src_subcommand in src_dict.values():
+        # Add all destination subcommands
+        for dest_subcommand in dest_dict.values():
+            src_subcommand.add_command(dest_subcommand)
+        cmd.add_command(src_subcommand)
+    return cmd

unstructured_ingest/cli/cmd_factory.py

@@ -0,0 +1,12 @@
+import typing as t
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.cmds import base_src_cmd_fns
+
+
+def get_src_cmd_map() -> t.Dict[str, t.Callable[[], BaseSrcCmd]]:
+    return {b().cmd_name_key: b for b in base_src_cmd_fns}
+
+
+def get_src_cmd(cmd_name: str) -> t.Callable[[], BaseSrcCmd]:
+    return get_src_cmd_map()[cmd_name]

unstructured_ingest/cli/cmds/__init__.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import collections
+import typing as t
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.cmds.fsspec.sftp import get_base_src_cmd as sftp_base_src_cmd
+
+from .airtable import get_base_src_cmd as airtable_base_src_cmd
+from .astradb import get_base_dest_cmd as astradb_base_dest_cmd
+from .astradb import get_base_src_cmd as astradb_base_src_cmd
+from .azure_ai_search import get_base_dest_cmd as azure_ai_search_base_dest_cmd
+from .biomed import get_base_src_cmd as biomed_base_src_cmd
+from .chroma import get_base_dest_cmd as chroma_base_dest_cmd
+from .clarifai import get_base_dest_cmd as clarifai_base_dest_cmd
+from .confluence import get_base_src_cmd as confluence_base_src_cmd
+from .databricks_volumes import get_base_dest_cmd as databricks_volumes_dest_cmd
+from .delta_table import get_base_dest_cmd as delta_table_dest_cmd
+from .delta_table import get_base_src_cmd as delta_table_base_src_cmd
+from .discord import get_base_src_cmd as discord_base_src_cmd
+from .elasticsearch import get_base_dest_cmd as elasticsearch_base_dest_cmd
+from .elasticsearch import get_base_src_cmd as elasticsearch_base_src_cmd
+from .fsspec.azure import get_base_dest_cmd as azure_base_dest_cmd
+from .fsspec.azure import get_base_src_cmd as azure_base_src_cmd
+from .fsspec.box import get_base_dest_cmd as box_base_dest_cmd
+from .fsspec.box import get_base_src_cmd as box_base_src_cmd
+from .fsspec.dropbox import get_base_dest_cmd as dropbox_base_dest_cmd
+from .fsspec.dropbox import get_base_src_cmd as dropbox_base_src_cmd
+from .fsspec.fsspec import get_base_dest_cmd as fsspec_base_dest_cmd
+from .fsspec.fsspec import get_base_src_cmd as fsspec_base_src_cmd
+from .fsspec.gcs import get_base_dest_cmd as gcs_base_dest_cmd
+from .fsspec.gcs import get_base_src_cmd as gcs_base_src_cmd
+from .fsspec.s3 import get_base_dest_cmd as s3_base_dest_cmd
+from .fsspec.s3 import get_base_src_cmd as s3_base_src_cmd
+from .github import get_base_src_cmd as github_base_src_cmd
+from .gitlab import get_base_src_cmd as gitlab_base_src_cmd
+from .google_drive import get_base_src_cmd as google_drive_base_src_cmd
+from .hubspot import get_base_src_cmd as hubspot_base_src_cmd
+from .jira import get_base_src_cmd as jira_base_src_cmd
+from .kafka import get_base_dest_cmd as kafka_base_dest_cmd
+from .kafka import get_base_src_cmd as kafka_base_src_cmd
+from .local import get_base_src_cmd as local_base_src_cmd
+from .mongodb import get_base_dest_cmd as mongo_base_dest_cmd
+from .mongodb import get_base_src_cmd as mongodb_base_src_cmd
+from .notion import get_base_src_cmd as notion_base_src_cmd
+from .onedrive import get_base_src_cmd as onedrive_base_src_cmd
+from .opensearch import get_base_dest_cmd as opensearch_base_dest_cmd
+from .opensearch import get_base_src_cmd as opensearch_base_src_cmd
+from .outlook import get_base_src_cmd as outlook_base_src_cmd
+from .pinecone import get_base_dest_cmd as pinecone_base_dest_cmd
+from .qdrant import get_base_dest_cmd as qdrant_base_dest_cmd
+from .reddit import get_base_src_cmd as reddit_base_src_cmd
+from .salesforce import get_base_src_cmd as salesforce_base_src_cmd
+from .sharepoint import get_base_src_cmd as sharepoint_base_src_cmd
+from .slack import get_base_src_cmd as slack_base_src_cmd
+from .sql import get_base_dest_cmd as sql_base_dest_cmd
+from .vectara import get_base_dest_cmd as vectara_base_dest_cmd
+from .weaviate import get_base_dest_cmd as weaviate_dest_cmd
+from .wikipedia import get_base_src_cmd as wikipedia_base_src_cmd
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+base_src_cmd_fns: t.List[t.Callable[[], BaseSrcCmd]] = [
+    airtable_base_src_cmd,
+    astradb_base_src_cmd,
+    azure_base_src_cmd,
+    biomed_base_src_cmd,
+    box_base_src_cmd,
+    confluence_base_src_cmd,
+    delta_table_base_src_cmd,
+    discord_base_src_cmd,
+    dropbox_base_src_cmd,
+    elasticsearch_base_src_cmd,
+    fsspec_base_src_cmd,
+    gcs_base_src_cmd,
+    github_base_src_cmd,
+    gitlab_base_src_cmd,
+    google_drive_base_src_cmd,
+    hubspot_base_src_cmd,
+    jira_base_src_cmd,
+    kafka_base_src_cmd,
+    local_base_src_cmd,
+    mongodb_base_src_cmd,
+    notion_base_src_cmd,
+    onedrive_base_src_cmd,
+    opensearch_base_src_cmd,
+    outlook_base_src_cmd,
+    reddit_base_src_cmd,
+    salesforce_base_src_cmd,
+    sftp_base_src_cmd,
+    sharepoint_base_src_cmd,
+    slack_base_src_cmd,
+    s3_base_src_cmd,
+    wikipedia_base_src_cmd,
+]
+
+# Make sure there are not overlapping names
+src_cmd_names = [b().cmd_name for b in base_src_cmd_fns]
+src_duplicates = [item for item, count in collections.Counter(src_cmd_names).items() if count > 1]
+if src_duplicates:
+    raise ValueError(
+        "multiple base src commands defined with the same names: {}".format(
+            ", ".join(src_duplicates),
+        ),
+    )
+
+base_dest_cmd_fns: t.List[t.Callable[[], "BaseDestCmd"]] = [
+    astradb_base_dest_cmd,
+    azure_base_dest_cmd,
+    box_base_dest_cmd,
+    chroma_base_dest_cmd,
+    clarifai_base_dest_cmd,
+    databricks_volumes_dest_cmd,
+    dropbox_base_dest_cmd,
+    elasticsearch_base_dest_cmd,
+    fsspec_base_dest_cmd,
+    gcs_base_dest_cmd,
+    kafka_base_dest_cmd,
+    s3_base_dest_cmd,
+    azure_ai_search_base_dest_cmd,
+    delta_table_dest_cmd,
+    sql_base_dest_cmd,
+    weaviate_dest_cmd,
+    mongo_base_dest_cmd,
+    pinecone_base_dest_cmd,
+    qdrant_base_dest_cmd,
+    opensearch_base_dest_cmd,
+    vectara_base_dest_cmd,
+]
+
+# Make sure there are not overlapping names
+dest_cmd_names = [b().cmd_name for b in base_dest_cmd_fns]
+dest_duplicates = [item for item, count in collections.Counter(dest_cmd_names).items() if count > 1]
+if dest_duplicates:
+    raise ValueError(
+        "multiple base dest commands defined with the same names: {}".format(
+            ", ".join(dest_duplicates),
+        ),
+    )
+
+__all__ = [
+    "base_src_cmd_fns",
+    "base_dest_cmd_fns",
+]

unstructured_ingest/cli/cmds/airtable.py

@@ -0,0 +1,69 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.airtable import SimpleAirtableConfig
+
+
+@dataclass
+class AirtableCliConfig(SimpleAirtableConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--personal-access-token"],
+                default=None,
+                help="Personal access token to authenticate into Airtable. Check: "
+                "https://support.airtable.com/docs/creating-and-using-api-keys-and-access-tokens "
+                "for more info",
+            ),
+            click.Option(
+                ["--list-of-paths"],
+                default=None,
+                help="""
+        A list of paths that specify the locations to ingest data from within Airtable.
+
+        If this argument is not set, the connector ingests all tables within each and every base.
+        --list-of-paths: path1 path2 path3 ….
+        path: base_id/table_id(optional)/view_id(optional)/
+
+        To obtain (base, table, view) ids in bulk, check:
+        https://airtable.com/developers/web/api/list-bases (base ids)
+        https://airtable.com/developers/web/api/get-base-schema (table and view ids)
+        https://pyairtable.readthedocs.io/en/latest/metadata.html (base, table and view ids)
+
+        To obtain specific ids from Airtable UI, go to your workspace, and copy any
+        relevant id from the URL structure:
+        https://airtable.com/appAbcDeF1ghijKlm/tblABcdEfG1HIJkLm/viwABCDEfg6hijKLM
+        appAbcDeF1ghijKlm -> base_id
+        tblABcdEfG1HIJkLm -> table_id
+        viwABCDEfg6hijKLM -> view_id
+
+        You can also check: https://support.airtable.com/docs/finding-airtable-ids
+
+        Here is an example for one --list-of-paths:
+            base1/		→ gets the entirety of all tables inside base1
+            base1/table1		→ gets all rows and columns within table1 in base1
+            base1/table1/view1	→ gets the rows and columns that are
+                                  visible in view1 for the table1 in base1
+
+        Examples to invalid airtable_paths:
+            table1          → has to mention base to be valid
+            base1/view1     → has to mention table to be valid
+                """,
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="airtable",
+        cli_config=AirtableCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/astradb.py

@@ -0,0 +1,99 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig, Dict
+from unstructured_ingest.connector.astradb import AstraDBWriteConfig, SimpleAstraDBConfig
+
+
+@dataclass
+class AstraDBCliConfig(SimpleAstraDBConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--token"],
+                required=True,
+                type=str,
+                help="Astra DB Token with access to the database.",
+                envvar="ASTRA_DB_APPLICATION_TOKEN",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--api-endpoint"],
+                required=True,
+                type=str,
+                help="The API endpoint for the Astra DB.",
+                envvar="ASTRA_DB_API_ENDPOINT",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--collection-name"],
+                required=False,
+                type=str,
+                help="The name of the Astra DB collection. "
+                "Note that the collection name must only include letters, "
+                "numbers, and underscores.",
+            ),
+            click.Option(
+                ["--keyspace"],
+                required=False,
+                default=None,
+                type=str,
+                help="The Astra DB connection keyspace.",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class AstraDBCliWriteConfig(AstraDBWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--embedding-dimension"],
+                required=True,
+                default=384,
+                type=int,
+                help="The dimensionality of the embeddings",
+            ),
+            click.Option(
+                ["--requested-indexing-policy"],
+                required=False,
+                default=None,
+                type=Dict(),
+                help="The indexing policy to use for the collection."
+                'example: \'{"deny": ["metadata"]}\' ',
+            ),
+            click.Option(
+                ["--batch-size"],
+                default=20,
+                type=int,
+                help="Number of records per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd():
+    from unstructured_ingest.cli.base.src import BaseSrcCmd
+
+    cmd_cls = BaseSrcCmd(
+        cmd_name="astradb",
+        cli_config=AstraDBCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="astradb",
+        cli_config=AstraDBCliConfig,
+        additional_cli_options=[AstraDBCliWriteConfig],
+        write_config=AstraDBWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/azure_ai_search.py

@@ -0,0 +1,65 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.azure_ai_search import (
+    AzureAISearchWriteConfig,
+    SimpleAzureAISearchStorageConfig,
+)
+
+
+@dataclass
+class AzureAISearchCliConfig(SimpleAzureAISearchStorageConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--key"],
+                required=True,
+                type=str,
+                help="Key credential used for authenticating to an Azure service.",
+                envvar="AZURE_SEARCH_API_KEY",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--endpoint"],
+                required=True,
+                type=str,
+                help="The URL endpoint of an Azure search service. "
+                "In the form of https://{{service_name}}.search.windows.net",
+                envvar="AZURE_SEARCH_ENDPOINT",
+                show_envvar=True,
+            ),
+        ]
+        return options
+
+
+@dataclass
+class AzureAISearchCliWriteConfig(AzureAISearchWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--index"],
+                required=True,
+                type=str,
+                help="The name of the index to connect to",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="azure-ai-search",
+        cli_config=AzureAISearchCliConfig,
+        additional_cli_options=[AzureAISearchCliWriteConfig],
+        write_config=AzureAISearchCliWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/biomed.py

@@ -0,0 +1,52 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.biomed import SimpleBiomedConfig
+
+
+@dataclass
+class BiomedCliConfig(SimpleBiomedConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-id"],
+                default=None,
+                help="ID parameter for OA Web Service API.",
+            ),
+            click.Option(
+                ["--api-from"],
+                default=None,
+                help="From parameter for OA Web Service API.",
+            ),
+            click.Option(
+                ["--api-until"],
+                default=None,
+                help="Until parameter for OA Web Service API.",
+            ),
+            click.Option(
+                ["--path"],
+                default=None,
+                help="PMC Open Access FTP Directory Path.",
+            ),
+            click.Option(
+                ["--max-request-time"],
+                default=45,
+                help="(In seconds) Max request time to OA Web Service API.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="biomed",
+        cli_config=BiomedCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/chroma.py

@@ -0,0 +1,104 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig, Dict
+from unstructured_ingest.connector.chroma import ChromaWriteConfig, SimpleChromaConfig
+
+
+@dataclass
+class ChromaCliConfig(SimpleChromaConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--path"],
+                required=False,
+                type=str,
+                help="Location where Chroma is persisted," "if not connecting via http.",
+            ),
+            click.Option(
+                ["--settings"],
+                required=False,
+                type=Dict(),
+                help="A dictionary of settings to communicate with the chroma server."
+                'example: \'{"persist_directory":"./chroma-persist"}\' ',
+            ),
+            click.Option(
+                ["--tenant"],
+                required=False,
+                default="default_tenant",
+                type=str,
+                help="The tenant to use for this client. Chroma defaults to 'default_tenant'.",
+            ),
+            click.Option(
+                ["--database"],
+                required=False,
+                default="default_database",
+                type=str,
+                help="The database to use for this client."
+                "Chroma defaults to 'default_database'.",
+            ),
+            click.Option(
+                ["--host"],
+                required=False,
+                type=str,
+                help="The hostname of the Chroma server.",
+            ),
+            click.Option(
+                ["--port"],
+                required=False,
+                type=int,
+                help="The port of the Chroma server.",
+            ),
+            click.Option(
+                ["--ssl"],
+                required=False,
+                default=False,
+                is_flag=True,
+                type=bool,
+                help="Whether to use SSL to connect to the Chroma server.",
+            ),
+            click.Option(
+                ["--headers"],
+                required=False,
+                type=Dict(),
+                help="A dictionary of headers to send to the Chroma server."
+                'example: \'{"Authorization":"Basic()"}\' ',
+            ),
+            click.Option(
+                ["--collection-name"],
+                required=True,
+                type=str,
+                help="The name of the Chroma collection to write into.",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class ChromaCliWriteConfig(ChromaWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=int,
+                help="Number of records per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="chroma",
+        cli_config=ChromaCliConfig,
+        additional_cli_options=[ChromaCliWriteConfig],
+        write_config=ChromaWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/clarifai.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.clarifai import (
+    ClarifaiWriteConfig,
+    SimpleClarifaiConfig,
+)
+
+CMD_NAME = "clarifai"
+
+
+@dataclass
+class ClarifaiCliConfig(SimpleClarifaiConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-key"],
+                required=True,
+                type=str,
+                help="The CLARIFAI_PAT of the user to access clarifai platform apps and models",
+                envvar="CLARIFAI_PAT",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--app-id"],
+                required=True,
+                type=str,
+                help="Clarifai app name/id",
+            ),
+            click.Option(
+                ["--user-id"],
+                required=True,
+                type=str,
+                help="Clarifai User name/ID",
+            ),
+            click.Option(
+                ["--dataset-id"], type=str, default=None, help="Clarifai App Dataset ID (optional)"
+            ),
+        ]
+        return options
+
+
+@dataclass
+class ClarifaiCliWriteConfig(ClarifaiWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                type=int,
+                default=50,
+                help="No of inputs upload per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=ClarifaiCliConfig,
+        additional_cli_options=[ClarifaiCliWriteConfig],
+        write_config=ClarifaiWriteConfig,
+    )
+    return cmd_cls