unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/cli/cmds/delta_table.py

@@ -1,94 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliConfig, Dict
-from unstructured_ingest.connector.delta_table import DeltaTableWriteConfig, SimpleDeltaTableConfig
-
-CMD_NAME = "delta-table"
-
-
-@dataclass
-class DeltaTableCliConfig(SimpleDeltaTableConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--table-uri"],
-                required=True,
-                help="the path of the DeltaTable",
-            ),
-            click.Option(
-                ["--version"],
-                default=None,
-                type=int,
-                help="version of the DeltaTable",
-            ),
-            click.Option(
-                ["--storage_options"],
-                required=False,
-                type=Dict(),
-                default=None,
-                help="a dictionary of the options to use for the storage backend, "
-                "passed in as a json string",
-            ),
-            click.Option(
-                ["--without-files"],
-                is_flag=True,
-                default=False,
-                help="If set, will load table without tracking files.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class DeltaTableCliWriteConfig(DeltaTableWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--overwrite-schema"],
-                is_flag=True,
-                default=False,
-                help="Flag to overwrite schema of destination table",
-            ),
-            click.Option(
-                ["--drop-empty-cols"],
-                is_flag=True,
-                default=False,
-                help="Flag to drop any columns that have no content",
-            ),
-            click.Option(
-                ["--mode"],
-                default="error",
-                type=click.Choice(["error", "append", "overwrite", "ignore"]),
-                help="How to handle existing data. Default is to error if table already exists. "
-                "If 'append', will add new data. "
-                "If 'overwrite', will replace table with new data. "
-                "If 'ignore', will not write anything if table already exists.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=DeltaTableCliConfig,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=DeltaTableCliConfig,
-        additional_cli_options=[DeltaTableCliWriteConfig],
-        write_config=DeltaTableWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/discord.py

@@ -1,47 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.discord import SimpleDiscordConfig
-
-
-@dataclass
-class DiscordCliConfig(SimpleDiscordConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--token"],
-                required=True,
-                help="Bot token used to access Discord API, must have "
-                "READ_MESSAGE_HISTORY scope for the bot user",
-            ),
-            click.Option(
-                ["--channels"],
-                required=True,
-                type=DelimitedString(),
-                help="Comma-delimited list of discord channel ids to ingest from.",
-            ),
-            click.Option(
-                ["--period"],
-                default=None,
-                type=click.IntRange(0),
-                help="Number of days to go back in the history of "
-                "discord channels, must be a number",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="discord",
-        cli_config=DiscordCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/elasticsearch.py

@@ -1,133 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
-from unstructured_ingest.connector.elasticsearch import (
-    ElasticsearchWriteConfig,
-    SimpleElasticsearchConfig,
-)
-
-CMD_NAME = "elasticsearch"
-
-
-@dataclass
-class ElasticsearchCliConfig(SimpleElasticsearchConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="Name of the Elasticsearch index to pull data from, or upload data to.",
-            ),
-            click.Option(
-                ["--hosts"],
-                type=DelimitedString(),
-                help='List of the Elasticsearch hosts to connect to, e.g. "http://localhost:9200"',
-            ),
-            click.Option(
-                ["--fields"],
-                type=DelimitedString(),
-                default=[],
-                help="If provided, will limit the fields returned by Elasticsearch "
-                "to this comma-delimited list",
-            ),
-            click.Option(
-                ["--username"], type=str, default=None, help="username when using basic auth"
-            ),
-            click.Option(
-                ["--password"],
-                type=str,
-                default=None,
-                help="password when using basic auth or connecting to a cloud instance",
-            ),
-            click.Option(
-                ["--cloud-id"], type=str, default=None, help="id used to connect to Elastic Cloud"
-            ),
-            click.Option(
-                ["--es-api-key"], type=str, default=None, help="api key used for authentication"
-            ),
-            click.Option(
-                ["--api-key-id"],
-                type=str,
-                default=None,
-                help="id associated with api key used for authentication: "
-                "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html",  # noqa: E501
-            ),
-            click.Option(
-                ["--bearer-auth"],
-                type=str,
-                default=None,
-                help="bearer token used for HTTP bearer authentication",
-            ),
-            click.Option(
-                ["--ca-certs"],
-                type=click.Path(),
-                default=None,
-            ),
-            click.Option(
-                ["--ssl-assert-fingerprint"],
-                type=str,
-                default=None,
-                help="SHA256 fingerprint value",
-            ),
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=click.IntRange(0),
-                help="how many records to read at a time per process",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ElasticsearchCliWriteConfig(ElasticsearchWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size-bytes"],
-                required=False,
-                default=15_000_000,
-                type=int,
-                help="Size limit (in bytes) for each batch of items to be uploaded. Check"
-                " https://www.elastic.co/guide/en/elasticsearch/guide/current/bulk.html"
-                "#_how_big_is_too_big for more information.",
-            ),
-            click.Option(
-                ["--num-processes"],
-                required=False,
-                default=1,
-                type=int,
-                help="Number of processes to be used while uploading content",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="elasticsearch",
-        cli_config=ElasticsearchCliConfig,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="elasticsearch",
-        cli_config=ElasticsearchCliConfig,
-        additional_cli_options=[ElasticsearchCliWriteConfig],
-        addition_configs={
-            "connector_config": SimpleElasticsearchConfig,
-            "write_config": ElasticsearchCliWriteConfig,
-        },
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/azure.py

@@ -1,94 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.fsspec.azure import (
-    AzureWriteConfig,
-    SimpleAzureBlobStorageConfig,
-)
-
-CMD_NAME = "azure"
-
-
-@dataclass
-class AzureCliConfig(SimpleAzureBlobStorageConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--account-key"],
-                default=None,
-                help="The storage account key. This is used for shared key "
-                "authentication. If any of account key, sas token or "
-                "client_id are not specified, anonymous access will be used.",
-            ),
-            click.Option(
-                ["--account-name"],
-                default=None,
-                help="The storage account name. This is used to authenticate "
-                "requests signed with an account key and to construct "
-                "the storage endpoint. It is required unless a connection "
-                "string is given, or if a custom domain is used with "
-                "anonymous authentication.",
-            ),
-            click.Option(
-                ["--connection-string"],
-                default=None,
-                help="If specified, this will override all other parameters. See "
-                "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
-                "for the connection string format.",
-            ),
-            click.Option(
-                ["--sas_token"],
-                default=None,
-                help="A shared access signature token to use to authenticate "
-                "requests instead of the account key. If account key and "
-                "sas token are both specified, account key will be used "
-                "to sign. If any of account key, sas token or client_id "
-                "are not specified, anonymous access will be used.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class AzureCliWriteConfig(AzureWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--overwrite"],
-                is_flag=True,
-                default=False,
-                show_default=True,
-                help="If set, will overwrite content if content already exists",
-            )
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=AzureCliConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=AzureCliConfig,
-        write_config=AzureCliWriteConfig,
-        is_fsspec=True,
-        additional_cli_options=[AzureCliWriteConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/box.py

@@ -1,48 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.fsspec.box import BoxWriteConfig, SimpleBoxConfig
-
-CMD_NAME = "box"
-
-
-@dataclass
-class BoxCliConfig(SimpleBoxConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--box-app-config"],
-                default=None,
-                type=click.Path(),
-                help="Path to Box app credentials as json file.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=BoxCliConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=BoxCliConfig,
-        write_config=BoxWriteConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/dropbox.py

@@ -1,51 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.fsspec.dropbox import (
-    DropboxWriteConfig,
-    SimpleDropboxConfig,
-)
-
-CMD_NAME = "dropbox"
-
-
-@dataclass
-class DropboxCliConfig(SimpleDropboxConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--token"],
-                required=True,
-                type=str,
-                help="Dropbox access token.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=DropboxCliConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=DropboxCliConfig,
-        write_config=DropboxWriteConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/fsspec.py

@@ -1,15 +0,0 @@
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-
-CMD_NAME = "fsspec"
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(cmd_name=CMD_NAME, is_fsspec=True)
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(cmd_name=CMD_NAME, is_fsspec=True)
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/gcs.py

@@ -1,71 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    FileOrJson,
-)
-from unstructured_ingest.connector.fsspec.gcs import GcsWriteConfig, SimpleGcsConfig
-
-CMD_NAME = "gcs"
-
-
-@dataclass
-class GcsCliConfig(SimpleGcsConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        help_string = """
-        Options:
-        - ``None``, GCSFS will attempt to guess your credentials in the
-      following order: gcloud CLI default, gcsfs cached token, google compute
-      metadata service, anonymous.
-        - ``'google_default'``, your default gcloud credentials will be used,
-          which are typically established by doing ``gcloud login`` in a terminal.
-        - ``'cache'``, credentials from previously successful gcsfs
-          authentication will be used (use this after "browser" auth succeeded)
-        - ``'anon'``, no authentication is performed, and you can only
-          access data which is accessible to allUsers (in this case, the project and
-          access level parameters are meaningless)
-        - ``'browser'``, you get an access code with which you can
-          authenticate via a specially provided URL
-        - if ``'cloud'``, we assume we are running within google compute
-          or google container engine, and query the internal metadata directly for
-          a token.
-        - you may supply a token generated by the
-          [gcloud](https://cloud.google.com/sdk/docs/)
-          utility; this is either a python dictionary or the name of a file
-          containing the JSON returned by logging in with the gcloud CLI tool.
-        """
-        options = [
-            click.Option(
-                ["--service-account-key"],
-                default=None,
-                type=FileOrJson(allow_raw_str=True),
-                help=help_string,
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=GcsCliConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=GcsCliConfig,
-        write_config=GcsWriteConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/s3.py

@@ -1,74 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.fsspec.s3 import S3WriteConfig, SimpleS3Config
-
-CMD_NAME = "s3"
-
-
-@dataclass
-class S3CliConfig(SimpleS3Config, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--anonymous"],
-                is_flag=True,
-                default=False,
-                help="Connect to s3 without local AWS credentials.",
-            ),
-            click.Option(
-                ["--endpoint-url"],
-                type=str,
-                default=None,
-                help="Use this endpoint_url, if specified. Needed for "
-                "connecting to non-AWS S3 buckets.",
-            ),
-            click.Option(
-                ["--key"],
-                type=str,
-                default=None,
-                help="If not anonymous, use this access key ID, if specified. Takes precedence "
-                "over `aws_access_key_id` in client_kwargs.",
-            ),
-            click.Option(
-                ["--secret"],
-                type=str,
-                default=None,
-                help="If not anonymous, use this secret access key, if specified.",
-            ),
-            click.Option(
-                ["--token"],
-                type=str,
-                default=None,
-                help="If not anonymous, use this security token, if specified.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd():
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=S3CliConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=S3CliConfig,
-        write_config=S3WriteConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/sftp.py

@@ -1,58 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.fsspec.sftp import SimpleSftpConfig
-
-CMD_NAME = "sftp"
-
-
-@dataclass
-class SftpCliConfig(SimpleSftpConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--username"],
-                required=True,
-                type=str,
-                help="Username for sftp connection",
-            ),
-            click.Option(
-                ["--password"],
-                required=True,
-                type=str,
-                help="Password for sftp connection",
-            ),
-            click.Option(
-                ["--look-for-keys"],
-                required=False,
-                default=False,
-                is_flag=True,
-                type=bool,
-                help="Whether to search for private key files in ~/.ssh/",
-            ),
-            click.Option(
-                ["--allow-agent"],
-                required=False,
-                default=False,
-                is_flag=True,
-                type=bool,
-                help="Whether to connect to the SSH agent.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=SftpCliConfig,
-        is_fsspec=True,
-    )
-    return cmd_cls