unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/cli/cmds/opensearch.py

@@ -1,117 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.cmds.elasticsearch import ElasticsearchCliWriteConfig
-from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
-from unstructured_ingest.connector.opensearch import SimpleOpenSearchConfig
-
-CMD_NAME = "opensearch"
-
-
-@dataclass
-class OpenSearchCliConfig(SimpleOpenSearchConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="Name of the OpenSearch index to pull data from, or upload data to.",
-            ),
-            click.Option(
-                ["--hosts"],
-                type=DelimitedString(),
-                help='List of the OpenSearch hosts to connect to, e.g. "http://localhost:9200"',
-            ),
-            click.Option(
-                ["--fields"],
-                type=DelimitedString(),
-                default=[],
-                help="If provided, will limit the fields returned by OpenSearch "
-                "to this comma-delimited list",
-            ),
-            click.Option(
-                ["--username"], type=str, default=None, help="username when using basic auth"
-            ),
-            click.Option(
-                ["--password"],
-                type=str,
-                default=None,
-                help="password when using basic auth",
-            ),
-            click.Option(
-                ["--use-ssl"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="use ssl for the connection",
-            ),
-            click.Option(
-                ["--verify-certs"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="whether to verify SSL certificates",
-            ),
-            click.Option(
-                ["--ssl-show-warn"],
-                type=bool,
-                default=False,
-                is_flag=True,
-                help="show warning when verify certs is disabled",
-            ),
-            click.Option(
-                ["--ca-certs"],
-                type=click.Path(),
-                default=None,
-                help="path to CA bundle",
-            ),
-            click.Option(
-                ["--client-cert"],
-                type=click.Path(),
-                default=None,
-                help="path to the file containing the private key and the certificate,"
-                " or cert only if using client_key",
-            ),
-            click.Option(
-                ["--client-key"],
-                type=click.Path(),
-                default=None,
-                help="path to the file containing the private key"
-                " if using separate cert and key files",
-            ),
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=click.IntRange(0),
-                help="how many records to read at a time per process",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="opensearch",
-        cli_config=OpenSearchCliConfig,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="opensearch",
-        cli_config=OpenSearchCliConfig,
-        additional_cli_options=[ElasticsearchCliWriteConfig],
-        addition_configs={
-            "connector_config": SimpleOpenSearchConfig,
-            "write_config": ElasticsearchCliWriteConfig,
-        },
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/outlook.py

@@ -1,67 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.outlook import SimpleOutlookConfig
-
-
-@dataclass
-class OutlookCliConfig(SimpleOutlookConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                required=True,
-                type=str,
-                help="Microsoft app client ID",
-            ),
-            click.Option(
-                ["--user-email"],
-                required=True,
-                type=str,
-                help="Outlook email to download messages from.",
-            ),
-            click.Option(
-                ["--tenant"],
-                default="common",
-                help="ID or domain name associated with your Azure AD instance",
-            ),
-            click.Option(
-                ["--outlook-folders"],
-                default=None,
-                type=DelimitedString(),
-                help="Folders to download email messages from. "
-                "Do not specify subfolders. Use quotes if spaces in folder names.",
-            ),
-            click.Option(
-                ["--client-cred"],
-                default=None,
-                type=str,
-                help="Microsoft App client secret",
-            ),
-            click.Option(
-                ["--authority-url"],
-                default="https://login.microsoftonline.com",
-                type=str,
-                help="Authentication token provider for Microsoft apps, default is "
-                "https://login.microsoftonline.com",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="outlook",
-        cli_config=OutlookCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/pinecone.py

@@ -1,71 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.pinecone import PineconeWriteConfig, SimplePineconeConfig
-
-
-@dataclass
-class PineconeCliConfig(SimplePineconeConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--api-key"],
-                required=True,
-                type=str,
-                help="API key used for authenticating to a Pinecone instance.",
-                envvar="PINECONE_API_KEY",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="The name of the pinecone index to connect to.",
-            ),
-            click.Option(
-                ["--environment"],
-                required=True,
-                type=str,
-                help="The environment where the index lives. Eg. 'gcp-starter' or 'us-east1-gcp'",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class PineconeCliWriteConfig(PineconeWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=50,
-                type=int,
-                help="Number of records per batch",
-            ),
-            click.Option(
-                ["--num-processes"],
-                default=2,
-                type=int,
-                help="Number of parallel processes with which to upload elements",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="pinecone",
-        cli_config=PineconeCliConfig,
-        additional_cli_options=[PineconeCliWriteConfig],
-        write_config=PineconeWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/qdrant.py

@@ -1,124 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.qdrant import QdrantWriteConfig, SimpleQdrantConfig
-
-
-@dataclass
-class QdrantCliConfig(SimpleQdrantConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--collection-name"],
-                required=True,
-                type=str,
-                help="The name of the Qdrant collection to use.",
-            ),
-            click.Option(
-                ["--location"],
-                type=str,
-                help="The location of the Qdrant cluster.",
-            ),
-            click.Option(
-                ["--url"],
-                type=str,
-                help="The location of the Qdrant cluster.",
-            ),
-            click.Option(
-                ["--port"],
-                type=int,
-                default=6333,
-                help="Port of the REST API interface. Default: 6333.",
-            ),
-            click.Option(
-                ["--grpc-port"],
-                type=int,
-                default=6334,
-                help="Port of the gRPC interface. Default: 6334.",
-            ),
-            click.Option(
-                ["--prefer-grpc"],
-                type=bool,
-                is_flag=True,
-                help="Whether to use gPRC interface whenever possible in methods. Default: False.",
-            ),
-            click.Option(
-                ["--https"],
-                type=bool,
-                is_flag=True,
-                help="Whether to use HTTPS(SSL) protocol. Default: False.",
-            ),
-            click.Option(
-                ["--prefix"],
-                type=str,
-                help="Prefix to add the REST API endpoints.",
-            ),
-            click.Option(
-                ["--timeout"],
-                type=int,
-                help="Timeout for operations. Default: 5.0 seconds for REST, unlimited for gRPC.",
-            ),
-            click.Option(
-                ["--host"],
-                type=str,
-                help="Host name of the Qdrant service.",
-            ),
-            click.Option(
-                ["--path"],
-                type=str,
-                help="Persistence path for QdrantLocal.",
-            ),
-            click.Option(
-                ["--force-disable-check-same-thread"],
-                type=bool,
-                is_flag=True,
-                help="Whether to force disable check same thread for QdrantLocal.",
-            ),
-            click.Option(
-                ["--api-key"],
-                type=str,
-                help="API key for authentication in Qdrant Cloud. Default: None.",
-                envvar="QDRANT_API_KEY",
-                show_envvar=True,
-            ),
-        ]
-        return options
-
-
-@dataclass
-class QdrantCliWriteConfig(QdrantWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=50,
-                type=int,
-                help="Number of points to upload per batch",
-            ),
-            click.Option(
-                ["--num-processes"],
-                default=2,
-                type=int,
-                help="Number of parallel processes with which to upload",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="qdrant",
-        cli_config=QdrantCliConfig,
-        additional_cli_options=[QdrantCliWriteConfig],
-        write_config=QdrantWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/reddit.py

@@ -1,67 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.reddit import SimpleRedditConfig
-
-
-@dataclass
-class RedditCliConfig(SimpleRedditConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                required=True,
-                type=str,
-                help="The client ID, see "
-                "https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#prerequisites"  # noqa: E501
-                " for more information.",
-            ),
-            click.Option(
-                ["--client-secret"],
-                required=True,
-                type=str,
-                help="The client secret, see "
-                "https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#prerequisites"  # noqa: E501
-                " for more information.",
-            ),
-            click.Option(
-                ["--subreddit-name"],
-                required=True,
-                type=str,
-                help='The name of a subreddit, without the "r\\", e.g. "machinelearning"',
-            ),
-            click.Option(
-                ["--search-query"],
-                default=None,
-                type=str,
-                help="If set, return posts using this query. Otherwise, use hot posts.",
-            ),
-            click.Option(
-                ["--num-posts"],
-                required=True,
-                type=click.IntRange(0),
-                help="If set, limits the number of posts to pull in.",
-            ),
-            click.Option(
-                ["--user-agent"],
-                required=True,
-                type=str,
-                help="user agent request header to use when calling Reddit API",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="reddit",
-        cli_config=RedditCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/salesforce.py

@@ -1,58 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.salesforce import SimpleSalesforceConfig
-
-
-@dataclass
-class SalesforceCliConfig(SimpleSalesforceConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        possible_categories = ["Account", "Case", "Campaign", "EmailMessage", "Lead"]
-        options = [
-            click.Option(
-                ["--username"],
-                required=True,
-                type=str,
-                help="Salesforce username usually looks like an email.",
-            ),
-            click.Option(
-                ["--consumer-key"],
-                required=True,
-                type=str,
-                help="For the Salesforce JWT auth. Found in Consumer Details.",
-            ),
-            click.Option(
-                ["--private-key"],
-                required=True,
-                type=str,
-                help="Path to the private key or its contents for the Salesforce JWT auth. "
-                "Key file is usually named server.key.",
-            ),
-            click.Option(
-                ["--categories"],
-                default=None,
-                required=True,
-                type=DelimitedString(choices=possible_categories),
-                help="Comma-delimited salesforce categories to download. "
-                "Currently only {}.".format(", ".join(possible_categories)),
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="salesforce",
-        cli_config=SalesforceCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/sharepoint.py

@@ -1,66 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-)
-from unstructured_ingest.connector.sharepoint import SimpleSharepointConfig
-
-
-@dataclass
-class SharepointCliConfig(SimpleSharepointConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                default=None,
-                type=str,
-                help="Sharepoint app client ID",
-            ),
-            click.Option(
-                ["--client-cred"],
-                default=None,
-                type=str,
-                help="Sharepoint app secret",
-            ),
-            click.Option(
-                ["--site"],
-                default=None,
-                type=str,
-                help="Sharepoint site url. Process either base url e.g \
-                    https://[tenant].sharepoint.com  or relative sites \
-                    https://[tenant].sharepoint.com/sites/<site_name>. \
-                    To process all sites within the tenant pass a site url as \
-                    https://[tenant]-admin.sharepoint.com.\
-                    This requires the app to be registered at a tenant level",
-            ),
-            click.Option(
-                ["--path"],
-                default="Shared Documents",
-                type=str,
-                help="Path from which to start parsing files. If the connector is to \
-                process all sites  within the tenant this filter will be applied to \
-                all sites document libraries. Default 'Shared Documents'",
-            ),
-            click.Option(
-                ["--files-only"],
-                is_flag=True,
-                default=False,
-                help="Process only files.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="sharepoint",
-        cli_config=SharepointCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/slack.py

@@ -1,56 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.slack import SimpleSlackConfig
-
-
-@dataclass
-class SlackCliConfig(SimpleSlackConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--token"],
-                required=True,
-                type=str,
-                help="Bot token used to access Slack API, must have channels:history "
-                "scope for the bot user",
-            ),
-            click.Option(
-                ["--channels"],
-                required=True,
-                type=DelimitedString(),
-                help="Comma-delimited list of Slack channel IDs to pull messages from, "
-                "can be a public or private channel",
-            ),
-            click.Option(
-                ["--start-date"],
-                default=None,
-                type=str,
-                help="Start date/time in formats YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or "
-                "YYYY-MM-DD+HH:MM:SS or YYYY-MM-DDTHH:MM:SStz",
-            ),
-            click.Option(
-                ["--end-date"],
-                default=None,
-                type=str,
-                help="End date/time in formats YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or "
-                "YYYY-MM-DD+HH:MM:SS or YYYY-MM-DDTHH:MM:SStz",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="slack",
-        cli_config=SlackCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/sql.py

@@ -1,66 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig
-from unstructured_ingest.connector.sql import SimpleSqlConfig
-from unstructured_ingest.interfaces import WriteConfig
-
-SQL_DRIVERS = {"postgresql", "sqlite"}
-
-
-@dataclass
-class SqlCliConfig(SimpleSqlConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--db-type"],
-                required=True,
-                type=click.Choice(SQL_DRIVERS),
-                help="Type of the database backend",
-            ),
-            click.Option(
-                ["--username"],
-                default=None,
-                type=str,
-                help="DB username",
-            ),
-            click.Option(
-                ["--password"],
-                default=None,
-                type=str,
-                help="DB password",
-            ),
-            click.Option(
-                ["--host"],
-                default=None,
-                type=str,
-                help="DB host",
-            ),
-            click.Option(
-                ["--port"],
-                default=None,
-                type=int,
-                help="DB host connection port",
-            ),
-            click.Option(
-                ["--database"],
-                default=None,
-                type=str,
-                help="Database name. For sqlite databases, this is the path to the .db file.",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="sql",
-        cli_config=SqlCliConfig,
-        write_config=WriteConfig,
-    )
-    return cmd_cls