unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/cli/cmds/opensearch.py

@@ -1,93 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.elasticsearch import (
-    ElasticsearchCliDownloadConfig,
-    ElasticsearchCliIndexerConfig,
-    ElasticsearchCliUploadStagerConfig,
-    ElasticsearchUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.opensearch import CONNECTOR_TYPE
-
-
-@dataclass
-class OpenSearchCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--hosts"],
-                type=DelimitedString(),
-                help='List of the OpenSearch hosts to connect to, e.g. "http://localhost:9200"',
-            ),
-            click.Option(
-                ["--username"], type=str, default=None, help="username when using basic auth"
-            ),
-            click.Option(
-                ["--password"],
-                type=str,
-                default=None,
-                help="password when using basic auth",
-            ),
-            click.Option(
-                ["--use-ssl"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="use ssl for the connection",
-            ),
-            click.Option(
-                ["--verify-certs"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="whether to verify SSL certificates",
-            ),
-            click.Option(
-                ["--ssl-show-warn"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="show warning when verify certs is disabled",
-            ),
-            click.Option(
-                ["--ca-certs"],
-                type=click.Path(),
-                default=None,
-                help="path to CA bundle",
-            ),
-            click.Option(
-                ["--client-cert"],
-                type=click.Path(),
-                default=None,
-                help="path to the file containing the private key and the certificate,"
-                " or cert only if using client_key",
-            ),
-            click.Option(
-                ["--client-key"],
-                type=click.Path(),
-                default=None,
-                help="path to the file containing the private key"
-                " if using separate cert and key files",
-            ),
-        ]
-        return options
-
-
-opensearch_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=OpenSearchCliConnectionConfig,
-    indexer_config=ElasticsearchCliIndexerConfig,
-    downloader_config=ElasticsearchCliDownloadConfig,
-)
-
-opensearch_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=OpenSearchCliConnectionConfig,
-    upload_stager_config=ElasticsearchCliUploadStagerConfig,
-    uploader_config=ElasticsearchUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/pinecone.py

@@ -1,62 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.pinecone import CONNECTOR_TYPE
-
-
-@dataclass
-class PineconeCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--api-key"],
-                required=True,
-                type=str,
-                help="API key for Pinecone.",
-            ),
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="Name of the index to connect to. Example: my-index",
-            ),
-            click.Option(
-                ["--environment"],
-                required=True,
-                type=str,
-                help="Environment to connect to. Example: us-east-1",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class PineconeCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            ),
-            click.Option(
-                ["--num-processes"],
-                default=4,
-                type=int,
-                help="Number of processes to use for uploading",
-            ),
-        ]
-        return options
-
-
-pinecone_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=PineconeCliConnectionConfig,
-    uploader_config=PineconeCliUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/salesforce.py

@@ -1,79 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.salesforce import (
-    ACCEPTED_CATEGORIES,
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class SalesforceCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--username"],
-                required=True,
-                type=str,
-                help="Salesforce username usually looks like an email.",
-            ),
-            click.Option(
-                ["--consumer-key"],
-                required=True,
-                type=str,
-                help="For the Salesforce JWT auth. Found in Consumer Details.",
-            ),
-            click.Option(
-                ["--private-key"],
-                required=True,
-                type=str,
-                help="Path to the private key or its contents for the Salesforce JWT auth. "
-                "Key file is usually named server.key.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SalesforceCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        possible_categories = ACCEPTED_CATEGORIES
-        options = [
-            click.Option(
-                ["--categories"],
-                default=None,
-                required=True,
-                type=DelimitedString(choices=possible_categories),
-                help="Comma-delimited salesforce categories to download. "
-                "Currently only {}.".format(", ".join(possible_categories)),
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SalesforceCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-        return options
-
-
-salesforce_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SalesforceCliConnectionConfig,
-    indexer_config=SalesforceCliIndexerConfig,
-    downloader_config=SalesforceCliDownloadConfig,
-)

unstructured_ingest/v2/cli/cmds/sharepoint.py

@@ -1,112 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.sharepoint import CONNECTOR_TYPE
-
-
-@dataclass
-class SharepointCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                default=None,
-                type=str,
-                help="Sharepoint app client ID",
-            ),
-            click.Option(
-                ["--client-cred"],
-                default=None,
-                type=str,
-                help="Sharepoint app secret",
-            ),
-            click.Option(
-                ["--site"],
-                default=None,
-                type=str,
-                help="Sharepoint site url. Process either base url e.g \
-                    https://[tenant].sharepoint.com  or relative sites \
-                    https://[tenant].sharepoint.com/sites/<site_name>. \
-                    To process all sites within the tenant pass a site url as \
-                    https://[tenant]-admin.sharepoint.com.\
-                    This requires the app to be registered at a tenant level",
-            ),
-            click.Option(
-                ["--permissions-application-id"],
-                type=str,
-                help="Microsoft Graph API application id",
-            ),
-            click.Option(
-                ["--permissions-client-cred"],
-                type=str,
-                help="Microsoft Graph API application credentials",
-            ),
-            click.Option(
-                ["--permissions-tenant"],
-                type=str,
-                help="e.g https://contoso.onmicrosoft.com to get permissions data within tenant.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SharepointCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--path"],
-                default=None,
-                type=str,
-                help="Path from which to start parsing files. If the connector is to \
-                process all sites within the tenant this filter will be applied to \
-                all sites document libraries.",
-            ),
-            click.Option(
-                ["--recursive"],
-                is_flag=True,
-                default=False,
-                help="Recursively download files in their respective folders "
-                "otherwise stop at the files in provided folder level.",
-            ),
-            click.Option(
-                ["--omit-files"],
-                is_flag=True,
-                default=False,
-                help="Don't process files.",
-            ),
-            click.Option(
-                ["--omit-pages"],
-                is_flag=True,
-                default=False,
-                help="Don't process site pages.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SharepointCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-        return options
-
-
-sharepoint_drive_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SharepointCliConnectionConfig,
-    indexer_config=SharepointCliIndexerConfig,
-    downloader_config=SharepointCliDownloadConfig,
-)

unstructured_ingest/v2/cli/cmds/singlestore.py

@@ -1,96 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.singlestore import CONNECTOR_TYPE
-
-
-@dataclass
-class SingleStoreCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--host"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore host",
-            ),
-            click.Option(
-                ["--port"],
-                required=False,
-                type=int,
-                default=None,
-                help="SingleStore port",
-            ),
-            click.Option(
-                ["--user"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore user",
-            ),
-            click.Option(
-                ["--password"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore password",
-            ),
-            click.Option(
-                ["--database"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore database",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SingleStoreCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--drop-empty-cols"],
-                required=False,
-                type=bool,
-                is_flag=True,
-                default=False,
-                help="Drop any columns that have no data",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SingleStoreCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return [
-            click.Option(
-                ["--table-name"],
-                required=False,
-                type=str,
-                help="SingleStore table to write contents to",
-            ),
-            click.Option(
-                ["--batch-size"],
-                required=False,
-                type=click.IntRange(min=1),
-                help="Batch size when writing to SingleStore",
-            ),
-        ]
-
-
-singlestore_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SingleStoreCliConnectionConfig,
-    uploader_config=SingleStoreCliUploaderConfig,
-    upload_stager_config=SingleStoreCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/sql.py

@@ -1,84 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.sql import CONNECTOR_TYPE
-
-SQL_DRIVERS = {"postgresql", "sqlite"}
-
-
-@dataclass
-class SQLCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--db-type"],
-                required=True,
-                type=click.Choice(SQL_DRIVERS),
-                help="Type of the database backend",
-            ),
-            click.Option(
-                ["--username"],
-                default=None,
-                type=str,
-                help="DB username",
-            ),
-            click.Option(
-                ["--password"],
-                default=None,
-                type=str,
-                help="DB password",
-            ),
-            click.Option(
-                ["--host"],
-                default=None,
-                type=str,
-                help="DB host",
-            ),
-            click.Option(
-                ["--port"],
-                default=None,
-                type=int,
-                help="DB host connection port",
-            ),
-            click.Option(
-                ["--database"],
-                default=None,
-                type=str,
-                help="Database name. For sqlite databases, this is the path to the .db file.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SQLCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-@dataclass
-class SQLCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-sql_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SQLCliConnectionConfig,
-    uploader_config=SQLCliUploaderConfig,
-    upload_stager_config=SQLCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/weaviate.py

@@ -1,100 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.weaviate import CONNECTOR_TYPE
-
-
-@dataclass
-class WeaviateCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--host-url"],
-                required=True,
-                help="Weaviate instance url",
-            ),
-            click.Option(
-                ["--class-name"],
-                default=None,
-                type=str,
-                help="Name of the class to push the records into, e.g: Pdf-elements",
-            ),
-            click.Option(
-                ["--access-token"], default=None, type=str, help="Used to create the bearer token."
-            ),
-            click.Option(
-                ["--refresh-token"],
-                default=None,
-                type=str,
-                help="Will tie this value to the bearer token. If not provided, "
-                "the authentication will expire once the lifetime of the access token is up.",
-            ),
-            click.Option(
-                ["--api-key"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--client-secret"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--scope"],
-                default=None,
-                type=DelimitedString(),
-            ),
-            click.Option(
-                ["--username"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--password"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--anonymous"],
-                is_flag=True,
-                default=False,
-                type=bool,
-                help="if set, all auth values will be ignored",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class WeaviateCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-@dataclass
-class WeaviateCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-weaviate_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=WeaviateCliConnectionConfig,
-    uploader_config=WeaviateCliUploaderConfig,
-    upload_stager_config=WeaviateCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/configs/__init__.py

@@ -1,6 +0,0 @@
-from .chunk import ChunkerCliConfig
-from .embed import EmbedderCliConfig
-from .partition import PartitionerCliConfig
-from .processor import ProcessorCliConfig
-
-__all__ = ["ChunkerCliConfig", "ProcessorCliConfig", "PartitionerCliConfig", "EmbedderCliConfig"]