unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

unstructured_ingest/v2/cli/cmds/chroma.py

@@ -1,108 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import Dict
-from unstructured_ingest.v2.processes.connectors.chroma import CONNECTOR_TYPE
-
-
-@dataclass
-class ChromaCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--path"],
-                required=False,
-                type=str,
-                help="Location where Chroma is persisted," "if not connecting via http.",
-            ),
-            click.Option(
-                ["--settings"],
-                required=False,
-                type=Dict(),
-                help="A dictionary of settings to communicate with the chroma server."
-                'example: \'{"persist_directory":"./chroma-persist"}\' ',
-            ),
-            click.Option(
-                ["--tenant"],
-                required=False,
-                default="default_tenant",
-                type=str,
-                help="The tenant to use for this client. Chroma defaults to 'default_tenant'.",
-            ),
-            click.Option(
-                ["--database"],
-                required=False,
-                default="default_database",
-                type=str,
-                help="The database to use for this client."
-                "Chroma defaults to 'default_database'.",
-            ),
-            click.Option(
-                ["--host"],
-                required=False,
-                type=str,
-                help="The hostname of the Chroma server.",
-            ),
-            click.Option(
-                ["--port"],
-                required=False,
-                type=int,
-                help="The port of the Chroma server.",
-            ),
-            click.Option(
-                ["--ssl"],
-                required=False,
-                default=False,
-                is_flag=True,
-                type=bool,
-                help="Whether to use SSL to connect to the Chroma server.",
-            ),
-            click.Option(
-                ["--headers"],
-                required=False,
-                type=Dict(),
-                help="A dictionary of headers to send to the Chroma server."
-                'example: \'{"Authorization":"Basic()"}\' ',
-            ),
-            click.Option(
-                ["--collection-name"],
-                required=True,
-                type=str,
-                help="The name of the Chroma collection to write into.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ChromaCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-@dataclass
-class ChromaCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-chroma_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=ChromaCliConnectionConfig,
-    uploader_config=ChromaCliUploaderConfig,
-    upload_stager_config=ChromaCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/databricks_volumes.py

@@ -1,161 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.databricks_volumes import CONNECTOR_TYPE
-
-
-@dataclass
-class DatabricksVolumesCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--host"],
-                type=str,
-                default=None,
-                help="The Databricks host URL for either the "
-                "Databricks workspace endpoint or the "
-                "Databricks accounts endpoint.",
-            ),
-            click.Option(
-                ["--account-id"],
-                type=str,
-                default=None,
-                help="The Databricks account ID for the Databricks "
-                "accounts endpoint. Only has effect when Host is "
-                "either https://accounts.cloud.databricks.com/ (AWS), "
-                "https://accounts.azuredatabricks.net/ (Azure), "
-                "or https://accounts.gcp.databricks.com/ (GCP).",
-            ),
-            click.Option(
-                ["--username"],
-                type=str,
-                default=None,
-                help="The Databricks username part of basic authentication. "
-                "Only possible when Host is *.cloud.databricks.com (AWS).",
-            ),
-            click.Option(
-                ["--password"],
-                type=str,
-                default=None,
-                help="The Databricks password part of basic authentication. "
-                "Only possible when Host is *.cloud.databricks.com (AWS).",
-            ),
-            click.Option(["--client-id"], type=str, default=None),
-            click.Option(["--client-secret"], type=str, default=None),
-            click.Option(
-                ["--token"],
-                type=str,
-                default=None,
-                help="The Databricks personal access token (PAT) (AWS, Azure, and GCP) or "
-                "Azure Active Directory (Azure AD) token (Azure).",
-            ),
-            click.Option(
-                ["--azure-workspace-resource-id"],
-                type=str,
-                default=None,
-                help="The Azure Resource Manager ID for the Azure Databricks workspace, "
-                "which is exchanged for a Databricks host URL.",
-            ),
-            click.Option(
-                ["--azure-client-secret"],
-                type=str,
-                default=None,
-                help="The Azure AD service principal’s client secret.",
-            ),
-            click.Option(
-                ["--azure-client-id"],
-                type=str,
-                default=None,
-                help="The Azure AD service principal’s application ID.",
-            ),
-            click.Option(
-                ["--azure-tenant-id"],
-                type=str,
-                default=None,
-                help="The Azure AD service principal’s tenant ID.",
-            ),
-            click.Option(
-                ["--azure-environment"],
-                type=str,
-                default=None,
-                help="The Azure environment type (such as Public, UsGov, China, and Germany) for a "
-                "specific set of API endpoints. Defaults to PUBLIC.",
-            ),
-            click.Option(
-                ["--auth-type"],
-                type=str,
-                default=None,
-                help="When multiple auth attributes are available in the "
-                "environment, use the auth type specified by this "
-                "argument. This argument also holds the currently "
-                "selected auth.",
-            ),
-            click.Option(["--cluster-id"], type=str, default=None),
-            click.Option(["--google-credentials"], type=str, default=None),
-            click.Option(["--google-service-account"], type=str, default=None),
-        ]
-        return options
-
-
-@dataclass
-class DatabricksVolumesCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--volume"], type=str, required=True, help="Name of volume in the Unity Catalog"
-            ),
-            click.Option(
-                ["--catalog"],
-                type=str,
-                required=True,
-                help="Name of the catalog in the Databricks Unity Catalog service",
-            ),
-            click.Option(
-                ["--volume-path"],
-                type=str,
-                required=False,
-                default=None,
-                help="Optional path within the volume to write to",
-            ),
-            click.Option(
-                ["--overwrite"],
-                type=bool,
-                is_flag=True,
-                help="If true, an existing file will be overwritten.",
-            ),
-            click.Option(
-                ["--encoding"],
-                type=str,
-                required=True,
-                default="utf-8",
-                help="Encoding applied to the data when written to the volume",
-            ),
-            click.Option(
-                ["--schema"],
-                type=str,
-                required=True,
-                default="default",
-                help="Schema associated with the volume to write to in the Unity Catalog service",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class DatabricksVolumesCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-databricks_volumes_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=DatabricksVolumesCliConnectionConfig,
-    uploader_config=DatabricksVolumesCliUploaderConfig,
-    upload_stager_config=DatabricksVolumesCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/elasticsearch.py

@@ -1,159 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.elasticsearch import CONNECTOR_TYPE
-
-
-@dataclass
-class ElasticsearchCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--hosts"],
-                type=DelimitedString(),
-                help='List of the Elasticsearch hosts to connect to, e.g. "http://localhost:9200"',
-            ),
-            click.Option(
-                ["--username"], type=str, default=None, help="username when using basic auth"
-            ),
-            click.Option(
-                ["--password"],
-                type=str,
-                default=None,
-                help="password when using basic auth or connecting to a cloud instance",
-            ),
-            click.Option(
-                ["--cloud-id"], type=str, default=None, help="id used to connect to Elastic Cloud"
-            ),
-            click.Option(
-                ["--es-api-key"], type=str, default=None, help="api key used for authentication"
-            ),
-            click.Option(
-                ["--api-key-id"],
-                type=str,
-                default=None,
-                help="id associated with api key used for authentication: "
-                "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html",  # noqa: E501
-                # noqa: E501
-            ),
-            click.Option(
-                ["--bearer-auth"],
-                type=str,
-                default=None,
-                help="bearer token used for HTTP bearer authentication",
-            ),
-            click.Option(
-                ["--ca-certs"],
-                type=click.Path(),
-                default=None,
-            ),
-            click.Option(
-                ["--ssl-assert-fingerprint"],
-                type=str,
-                default=None,
-                help="SHA256 fingerprint value",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ElasticsearchCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-            click.Option(
-                ["--fields"],
-                type=DelimitedString(),
-                default=[],
-                help="If provided, will limit the fields returned by Elasticsearch "
-                "to this comma-delimited list",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ElasticsearchCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="Name of the Elasticsearch index to pull data from, or upload data to.",
-            ),
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=click.IntRange(0),
-                help="how many records to read at a time per process",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ElasticsearchCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="Name of the Elasticsearch index to pull data from, or upload data to.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ElasticsearchUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size-bytes"],
-                required=False,
-                default=15_000_000,
-                type=int,
-                help="Size limit (in bytes) for each batch of items to be uploaded. Check"
-                " https://www.elastic.co/guide/en/elasticsearch/guide/current/bulk.html"
-                "#_how_big_is_too_big for more information.",
-            ),
-            click.Option(
-                ["--num-threads"],
-                required=False,
-                default=1,
-                type=int,
-                help="Number of threads to be used while uploading content",
-            ),
-        ]
-        return options
-
-
-elasticsearch_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=ElasticsearchCliConnectionConfig,
-    indexer_config=ElasticsearchCliIndexerConfig,
-    downloader_config=ElasticsearchCliDownloadConfig,
-)
-
-elasticsearch_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=ElasticsearchCliConnectionConfig,
-    upload_stager_config=ElasticsearchCliUploadStagerConfig,
-    uploader_config=ElasticsearchUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/fsspec/azure.py

@@ -1,84 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.fsspec.fsspec import (
-    FsspecCliDownloadConfig,
-    FsspecCliIndexerConfig,
-    FsspecCliUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.fsspec.azure import (
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class AzureCliDownloadConfig(FsspecCliDownloadConfig):
-    pass
-
-
-@dataclass
-class AzureCliIndexerConfig(FsspecCliIndexerConfig):
-    pass
-
-
-@dataclass
-class AzureCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--account-key"],
-                default=None,
-                help="The storage account key. This is used for shared key "
-                "authentication. If any of account key, sas token or "
-                "client_id are not specified, anonymous access will be used.",
-            ),
-            click.Option(
-                ["--account-name"],
-                default=None,
-                help="The storage account name. This is used to authenticate "
-                "requests signed with an account key and to construct "
-                "the storage endpoint. It is required unless a connection "
-                "string is given, or if a custom domain is used with "
-                "anonymous authentication.",
-            ),
-            click.Option(
-                ["--connection-string"],
-                default=None,
-                help="If specified, this will override all other parameters. See "
-                "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
-                "for the connection string format.",
-            ),
-            click.Option(
-                ["--sas_token"],
-                default=None,
-                help="A shared access signature token to use to authenticate "
-                "requests instead of the account key. If account key and "
-                "sas token are both specified, account key will be used "
-                "to sign. If any of account key, sas token or client_id "
-                "are not specified, anonymous access will be used.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class AzureUploaderConfig(FsspecCliUploaderConfig):
-    pass
-
-
-azure_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=AzureCliIndexerConfig,
-    connection_config=AzureCliConnectionConfig,
-    downloader_config=AzureCliDownloadConfig,
-)
-
-azure_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=AzureCliConnectionConfig,
-    uploader_config=AzureUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/fsspec/box.py

@@ -1,58 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.fsspec.fsspec import (
-    FsspecCliDownloadConfig,
-    FsspecCliIndexerConfig,
-    FsspecCliUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.fsspec.box import (
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class BoxCliDownloadConfig(FsspecCliDownloadConfig):
-    pass
-
-
-@dataclass
-class BoxCliIndexerConfig(FsspecCliIndexerConfig):
-    pass
-
-
-@dataclass
-class BoxCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--box-app-config"],
-                default=None,
-                type=click.Path(),
-                help="Path to Box app credentials as json file.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class BoxUploaderConfig(FsspecCliUploaderConfig):
-    pass
-
-
-box_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=BoxCliIndexerConfig,
-    connection_config=BoxCliConnectionConfig,
-    downloader_config=BoxCliDownloadConfig,
-)
-
-box_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=BoxCliConnectionConfig,
-    uploader_config=BoxUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/fsspec/dropbox.py

@@ -1,58 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.fsspec.fsspec import (
-    FsspecCliDownloadConfig,
-    FsspecCliIndexerConfig,
-    FsspecCliUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.fsspec.dropbox import (
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class DropboxCliDownloadConfig(FsspecCliDownloadConfig):
-    pass
-
-
-@dataclass
-class DropboxCliIndexerConfig(FsspecCliIndexerConfig):
-    pass
-
-
-@dataclass
-class DropboxCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--token"],
-                required=True,
-                type=str,
-                help="Dropbox access token.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class DropboxUploaderConfig(FsspecCliUploaderConfig):
-    pass
-
-
-dropbox_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=DropboxCliIndexerConfig,
-    connection_config=DropboxCliConnectionConfig,
-    downloader_config=DropboxCliDownloadConfig,
-)
-
-dropbox_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=DropboxCliConnectionConfig,
-    uploader_config=DropboxUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/fsspec/fsspec.py

@@ -1,69 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-
-
-@dataclass
-class FsspecCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured/ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-
-
-@dataclass
-class FsspecCliFileConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return [
-            click.Option(
-                ["--remote-url"],
-                required=True,
-                help="Remote fsspec URL formatted as `protocol://dir/path`",
-            )
-        ]
-
-
-@dataclass
-class FsspecCliUploaderConfig(FsspecCliFileConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = super(FsspecCliUploaderConfig, FsspecCliUploaderConfig).get_cli_options()
-        options.extend(
-            [
-                click.Option(
-                    ["--overwrite"],
-                    is_flag=True,
-                    default=False,
-                    show_default=True,
-                    help="If set, will overwrite content if content already exists",
-                )
-            ]
-        )
-        return options
-
-
-@dataclass
-class FsspecCliIndexerConfig(FsspecCliFileConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = super(FsspecCliIndexerConfig, FsspecCliIndexerConfig).get_cli_options()
-        options.extend(
-            [
-                click.Option(
-                    ["--recursive"],
-                    is_flag=True,
-                    default=False,
-                    help="Recursively download files in their respective folders "
-                    "otherwise stop at the files in provided folder level.",
-                ),
-            ]
-        )
-        return options