unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/cli/cmds/chroma.py

@@ -0,0 +1,104 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig, Dict
+from unstructured_ingest.connector.chroma import ChromaWriteConfig, SimpleChromaConfig
+
+
+@dataclass
+class ChromaCliConfig(SimpleChromaConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--path"],
+                required=False,
+                type=str,
+                help="Location where Chroma is persisted," "if not connecting via http.",
+            ),
+            click.Option(
+                ["--settings"],
+                required=False,
+                type=Dict(),
+                help="A dictionary of settings to communicate with the chroma server."
+                'example: \'{"persist_directory":"./chroma-persist"}\' ',
+            ),
+            click.Option(
+                ["--tenant"],
+                required=False,
+                default="default_tenant",
+                type=str,
+                help="The tenant to use for this client. Chroma defaults to 'default_tenant'.",
+            ),
+            click.Option(
+                ["--database"],
+                required=False,
+                default="default_database",
+                type=str,
+                help="The database to use for this client."
+                "Chroma defaults to 'default_database'.",
+            ),
+            click.Option(
+                ["--host"],
+                required=False,
+                type=str,
+                help="The hostname of the Chroma server.",
+            ),
+            click.Option(
+                ["--port"],
+                required=False,
+                type=int,
+                help="The port of the Chroma server.",
+            ),
+            click.Option(
+                ["--ssl"],
+                required=False,
+                default=False,
+                is_flag=True,
+                type=bool,
+                help="Whether to use SSL to connect to the Chroma server.",
+            ),
+            click.Option(
+                ["--headers"],
+                required=False,
+                type=Dict(),
+                help="A dictionary of headers to send to the Chroma server."
+                'example: \'{"Authorization":"Basic()"}\' ',
+            ),
+            click.Option(
+                ["--collection-name"],
+                required=True,
+                type=str,
+                help="The name of the Chroma collection to write into.",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class ChromaCliWriteConfig(ChromaWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=int,
+                help="Number of records per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="chroma",
+        cli_config=ChromaCliConfig,
+        additional_cli_options=[ChromaCliWriteConfig],
+        write_config=ChromaWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/clarifai.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.clarifai import (
+    ClarifaiWriteConfig,
+    SimpleClarifaiConfig,
+)
+
+CMD_NAME = "clarifai"
+
+
+@dataclass
+class ClarifaiCliConfig(SimpleClarifaiConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-key"],
+                required=True,
+                type=str,
+                help="The CLARIFAI_PAT of the user to access clarifai platform apps and models",
+                envvar="CLARIFAI_PAT",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--app-id"],
+                required=True,
+                type=str,
+                help="Clarifai app name/id",
+            ),
+            click.Option(
+                ["--user-id"],
+                required=True,
+                type=str,
+                help="Clarifai User name/ID",
+            ),
+            click.Option(
+                ["--dataset-id"], type=str, default=None, help="Clarifai App Dataset ID (optional)"
+            ),
+        ]
+        return options
+
+
+@dataclass
+class ClarifaiCliWriteConfig(ClarifaiWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                type=int,
+                default=50,
+                help="No of inputs upload per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=ClarifaiCliConfig,
+        additional_cli_options=[ClarifaiCliWriteConfig],
+        write_config=ClarifaiWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/confluence.py

@@ -0,0 +1,69 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.confluence import SimpleConfluenceConfig
+
+
+@dataclass
+class ConfluenceCliConfig(SimpleConfluenceConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-token"],
+                required=True,
+                help="API Token to authenticate into Confluence Cloud. "
+                "Check "
+                "https://developer.atlassian.com/cloud/confluence/basic-auth-for-rest-apis/ "
+                "for more info.",
+            ),
+            click.Option(
+                ["--url"],
+                required=True,
+                help='URL to Confluence Cloud, e.g. "unstructured-ingest-test.atlassian.net"',
+            ),
+            click.Option(
+                ["--user-email"],
+                required=True,
+                help="Email to authenticate into Confluence Cloud",
+            ),
+            click.Option(
+                ["--spaces"],
+                default=None,
+                type=DelimitedString(),
+                help="A list of confluence space ids to be fetched. From each fetched space, "
+                "--num-of-docs-from-each-space number of docs will be ingested. "
+                "--spaces and --num-of-spaces cannot be used at the same time",
+            ),
+            click.Option(
+                ["--max-num-of-docs-from-each-space"],
+                default=100,
+                help="Number of documents to be aimed to be ingested from each fetched "
+                "confluence space. If any space has fewer documents, all the documents from "
+                "that space will be ingested. Documents are not necessarily "
+                "ingested in order of creation date.",
+            ),
+            click.Option(
+                ["--max-num-of-spaces"],
+                default=500,
+                help="Number of confluence space ids to be fetched. From each fetched space, "
+                "--num-of-docs-from-each-space number of docs will be ingested. "
+                "--spaces and --num-of-spaces cannot be used at the same time",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="confluence",
+        cli_config=ConfluenceCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/databricks_volumes.py

@@ -0,0 +1,163 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.databricks_volumes import (
+    DatabricksVolumesWriteConfig,
+    SimpleDatabricksVolumesConfig,
+)
+
+CMD_NAME = "databricks-volumes"
+
+
+@dataclass
+class DatabricksVolumesCliConfig(SimpleDatabricksVolumesConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--host"],
+                type=str,
+                default=None,
+                help="The Databricks host URL for either the "
+                "Databricks workspace endpoint or the "
+                "Databricks accounts endpoint.",
+            ),
+            click.Option(
+                ["--account-id"],
+                type=str,
+                default=None,
+                help="The Databricks account ID for the Databricks "
+                "accounts endpoint. Only has effect when Host is "
+                "either https://accounts.cloud.databricks.com/ (AWS), "
+                "https://accounts.azuredatabricks.net/ (Azure), "
+                "or https://accounts.gcp.databricks.com/ (GCP).",
+            ),
+            click.Option(
+                ["--username"],
+                type=str,
+                default=None,
+                help="The Databricks username part of basic authentication. "
+                "Only possible when Host is *.cloud.databricks.com (AWS).",
+            ),
+            click.Option(
+                ["--password"],
+                type=str,
+                default=None,
+                help="The Databricks password part of basic authentication. "
+                "Only possible when Host is *.cloud.databricks.com (AWS).",
+            ),
+            click.Option(["--client-id"], type=str, default=None),
+            click.Option(["--client-secret"], type=str, default=None),
+            click.Option(
+                ["--token"],
+                type=str,
+                default=None,
+                help="The Databricks personal access token (PAT) (AWS, Azure, and GCP) or "
+                "Azure Active Directory (Azure AD) token (Azure).",
+            ),
+            click.Option(
+                ["--azure-workspace-resource-id"],
+                type=str,
+                default=None,
+                help="The Azure Resource Manager ID for the Azure Databricks workspace, "
+                "which is exchanged for a Databricks host URL.",
+            ),
+            click.Option(
+                ["--azure-client-secret"],
+                type=str,
+                default=None,
+                help="The Azure AD service principal’s client secret.",
+            ),
+            click.Option(
+                ["--azure-client-id"],
+                type=str,
+                default=None,
+                help="The Azure AD service principal’s application ID.",
+            ),
+            click.Option(
+                ["--azure-tenant-id"],
+                type=str,
+                default=None,
+                help="The Azure AD service principal’s tenant ID.",
+            ),
+            click.Option(
+                ["--azure-environment"],
+                type=str,
+                default=None,
+                help="The Azure environment type (such as Public, UsGov, China, and Germany) for a "
+                "specific set of API endpoints. Defaults to PUBLIC.",
+            ),
+            click.Option(
+                ["--auth-type"],
+                type=str,
+                default=None,
+                help="When multiple auth attributes are available in the "
+                "environment, use the auth type specified by this "
+                "argument. This argument also holds the currently "
+                "selected auth.",
+            ),
+            click.Option(["--cluster-id"], type=str, default=None),
+            click.Option(["--google-credentials"], type=str, default=None),
+            click.Option(["--google-service-account"], type=str, default=None),
+        ]
+        return options
+
+
+@dataclass
+class DatabricksVolumesCliWriteConfig(DatabricksVolumesWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--volume"], type=str, required=True, help="Name of volume in the Unity Catalog"
+            ),
+            click.Option(
+                ["--catalog"],
+                type=str,
+                required=True,
+                help="Name of the catalog in the Databricks Unity Catalog service",
+            ),
+            click.Option(
+                ["--volume-path"],
+                type=str,
+                required=False,
+                default=None,
+                help="Optional path within the volume to write to",
+            ),
+            click.Option(
+                ["--overwrite"],
+                type=bool,
+                is_flag=True,
+                help="If true, an existing file will be overwritten.",
+            ),
+            click.Option(
+                ["--encoding"],
+                type=str,
+                required=True,
+                default="utf-8",
+                help="Encoding applied to the data when written to the volume",
+            ),
+            click.Option(
+                ["--schema"],
+                type=str,
+                required=True,
+                default="default",
+                help="Schema associated with the volume to write to in the Unity Catalog service",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=DatabricksVolumesCliConfig,
+        additional_cli_options=[DatabricksVolumesCliWriteConfig],
+        write_config=DatabricksVolumesWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/delta_table.py

@@ -0,0 +1,94 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig, Dict
+from unstructured_ingest.connector.delta_table import DeltaTableWriteConfig, SimpleDeltaTableConfig
+
+CMD_NAME = "delta-table"
+
+
+@dataclass
+class DeltaTableCliConfig(SimpleDeltaTableConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--table-uri"],
+                required=True,
+                help="the path of the DeltaTable",
+            ),
+            click.Option(
+                ["--version"],
+                default=None,
+                type=int,
+                help="version of the DeltaTable",
+            ),
+            click.Option(
+                ["--storage_options"],
+                required=False,
+                type=Dict(),
+                default=None,
+                help="a dictionary of the options to use for the storage backend, "
+                "passed in as a json string",
+            ),
+            click.Option(
+                ["--without-files"],
+                is_flag=True,
+                default=False,
+                help="If set, will load table without tracking files.",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class DeltaTableCliWriteConfig(DeltaTableWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--overwrite-schema"],
+                is_flag=True,
+                default=False,
+                help="Flag to overwrite schema of destination table",
+            ),
+            click.Option(
+                ["--drop-empty-cols"],
+                is_flag=True,
+                default=False,
+                help="Flag to drop any columns that have no content",
+            ),
+            click.Option(
+                ["--mode"],
+                default="error",
+                type=click.Choice(["error", "append", "overwrite", "ignore"]),
+                help="How to handle existing data. Default is to error if table already exists. "
+                "If 'append', will add new data. "
+                "If 'overwrite', will replace table with new data. "
+                "If 'ignore', will not write anything if table already exists.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=DeltaTableCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=DeltaTableCliConfig,
+        additional_cli_options=[DeltaTableCliWriteConfig],
+        write_config=DeltaTableWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/discord.py

@@ -0,0 +1,47 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.discord import SimpleDiscordConfig
+
+
+@dataclass
+class DiscordCliConfig(SimpleDiscordConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--token"],
+                required=True,
+                help="Bot token used to access Discord API, must have "
+                "READ_MESSAGE_HISTORY scope for the bot user",
+            ),
+            click.Option(
+                ["--channels"],
+                required=True,
+                type=DelimitedString(),
+                help="Comma-delimited list of discord channel ids to ingest from.",
+            ),
+            click.Option(
+                ["--period"],
+                default=None,
+                type=click.IntRange(0),
+                help="Number of days to go back in the history of "
+                "discord channels, must be a number",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="discord",
+        cli_config=DiscordCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/elasticsearch.py

@@ -0,0 +1,133 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.elasticsearch import (
+    ElasticsearchWriteConfig,
+    SimpleElasticsearchConfig,
+)
+
+CMD_NAME = "elasticsearch"
+
+
+@dataclass
+class ElasticsearchCliConfig(SimpleElasticsearchConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--index-name"],
+                required=True,
+                type=str,
+                help="Name of the Elasticsearch index to pull data from, or upload data to.",
+            ),
+            click.Option(
+                ["--hosts"],
+                type=DelimitedString(),
+                help='List of the Elasticsearch hosts to connect to, e.g. "http://localhost:9200"',
+            ),
+            click.Option(
+                ["--fields"],
+                type=DelimitedString(),
+                default=[],
+                help="If provided, will limit the fields returned by Elasticsearch "
+                "to this comma-delimited list",
+            ),
+            click.Option(
+                ["--username"], type=str, default=None, help="username when using basic auth"
+            ),
+            click.Option(
+                ["--password"],
+                type=str,
+                default=None,
+                help="password when using basic auth or connecting to a cloud instance",
+            ),
+            click.Option(
+                ["--cloud-id"], type=str, default=None, help="id used to connect to Elastic Cloud"
+            ),
+            click.Option(
+                ["--es-api-key"], type=str, default=None, help="api key used for authentication"
+            ),
+            click.Option(
+                ["--api-key-id"],
+                type=str,
+                default=None,
+                help="id associated with api key used for authentication: "
+                "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html",  # noqa: E501
+            ),
+            click.Option(
+                ["--bearer-auth"],
+                type=str,
+                default=None,
+                help="bearer token used for HTTP bearer authentication",
+            ),
+            click.Option(
+                ["--ca-certs"],
+                type=click.Path(),
+                default=None,
+            ),
+            click.Option(
+                ["--ssl-assert-fingerprint"],
+                type=str,
+                default=None,
+                help="SHA256 fingerprint value",
+            ),
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=click.IntRange(0),
+                help="how many records to read at a time per process",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class ElasticsearchCliWriteConfig(ElasticsearchWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size-bytes"],
+                required=False,
+                default=15_000_000,
+                type=int,
+                help="Size limit (in bytes) for each batch of items to be uploaded. Check"
+                " https://www.elastic.co/guide/en/elasticsearch/guide/current/bulk.html"
+                "#_how_big_is_too_big for more information.",
+            ),
+            click.Option(
+                ["--num-processes"],
+                required=False,
+                default=1,
+                type=int,
+                help="Number of processes to be used while uploading content",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="elasticsearch",
+        cli_config=ElasticsearchCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="elasticsearch",
+        cli_config=ElasticsearchCliConfig,
+        additional_cli_options=[ElasticsearchCliWriteConfig],
+        addition_configs={
+            "connector_config": SimpleElasticsearchConfig,
+            "write_config": ElasticsearchCliWriteConfig,
+        },
+    )
+    return cmd_cls