unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/cli/cmds/qdrant.py

@@ -0,0 +1,124 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.qdrant import QdrantWriteConfig, SimpleQdrantConfig
+
+
+@dataclass
+class QdrantCliConfig(SimpleQdrantConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--collection-name"],
+                required=True,
+                type=str,
+                help="The name of the Qdrant collection to use.",
+            ),
+            click.Option(
+                ["--location"],
+                type=str,
+                help="The location of the Qdrant cluster.",
+            ),
+            click.Option(
+                ["--url"],
+                type=str,
+                help="The location of the Qdrant cluster.",
+            ),
+            click.Option(
+                ["--port"],
+                type=int,
+                default=6333,
+                help="Port of the REST API interface. Default: 6333.",
+            ),
+            click.Option(
+                ["--grpc-port"],
+                type=int,
+                default=6334,
+                help="Port of the gRPC interface. Default: 6334.",
+            ),
+            click.Option(
+                ["--prefer-grpc"],
+                type=bool,
+                is_flag=True,
+                help="Whether to use gPRC interface whenever possible in methods. Default: False.",
+            ),
+            click.Option(
+                ["--https"],
+                type=bool,
+                is_flag=True,
+                help="Whether to use HTTPS(SSL) protocol. Default: False.",
+            ),
+            click.Option(
+                ["--prefix"],
+                type=str,
+                help="Prefix to add the REST API endpoints.",
+            ),
+            click.Option(
+                ["--timeout"],
+                type=int,
+                help="Timeout for operations. Default: 5.0 seconds for REST, unlimited for gRPC.",
+            ),
+            click.Option(
+                ["--host"],
+                type=str,
+                help="Host name of the Qdrant service.",
+            ),
+            click.Option(
+                ["--path"],
+                type=str,
+                help="Persistence path for QdrantLocal.",
+            ),
+            click.Option(
+                ["--force-disable-check-same-thread"],
+                type=bool,
+                is_flag=True,
+                help="Whether to force disable check same thread for QdrantLocal.",
+            ),
+            click.Option(
+                ["--api-key"],
+                type=str,
+                help="API key for authentication in Qdrant Cloud. Default: None.",
+                envvar="QDRANT_API_KEY",
+                show_envvar=True,
+            ),
+        ]
+        return options
+
+
+@dataclass
+class QdrantCliWriteConfig(QdrantWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=50,
+                type=int,
+                help="Number of points to upload per batch",
+            ),
+            click.Option(
+                ["--num-processes"],
+                default=2,
+                type=int,
+                help="Number of parallel processes with which to upload",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="qdrant",
+        cli_config=QdrantCliConfig,
+        additional_cli_options=[QdrantCliWriteConfig],
+        write_config=QdrantWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/reddit.py

@@ -0,0 +1,67 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.reddit import SimpleRedditConfig
+
+
+@dataclass
+class RedditCliConfig(SimpleRedditConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                required=True,
+                type=str,
+                help="The client ID, see "
+                "https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#prerequisites"  # noqa: E501
+                " for more information.",
+            ),
+            click.Option(
+                ["--client-secret"],
+                required=True,
+                type=str,
+                help="The client secret, see "
+                "https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#prerequisites"  # noqa: E501
+                " for more information.",
+            ),
+            click.Option(
+                ["--subreddit-name"],
+                required=True,
+                type=str,
+                help='The name of a subreddit, without the "r\\", e.g. "machinelearning"',
+            ),
+            click.Option(
+                ["--search-query"],
+                default=None,
+                type=str,
+                help="If set, return posts using this query. Otherwise, use hot posts.",
+            ),
+            click.Option(
+                ["--num-posts"],
+                required=True,
+                type=click.IntRange(0),
+                help="If set, limits the number of posts to pull in.",
+            ),
+            click.Option(
+                ["--user-agent"],
+                required=True,
+                type=str,
+                help="user agent request header to use when calling Reddit API",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="reddit",
+        cli_config=RedditCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/salesforce.py

@@ -0,0 +1,58 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.salesforce import SimpleSalesforceConfig
+
+
+@dataclass
+class SalesforceCliConfig(SimpleSalesforceConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        possible_categories = ["Account", "Case", "Campaign", "EmailMessage", "Lead"]
+        options = [
+            click.Option(
+                ["--username"],
+                required=True,
+                type=str,
+                help="Salesforce username usually looks like an email.",
+            ),
+            click.Option(
+                ["--consumer-key"],
+                required=True,
+                type=str,
+                help="For the Salesforce JWT auth. Found in Consumer Details.",
+            ),
+            click.Option(
+                ["--private-key"],
+                required=True,
+                type=str,
+                help="Path to the private key or its contents for the Salesforce JWT auth. "
+                "Key file is usually named server.key.",
+            ),
+            click.Option(
+                ["--categories"],
+                default=None,
+                required=True,
+                type=DelimitedString(choices=possible_categories),
+                help="Comma-delimited salesforce categories to download. "
+                "Currently only {}.".format(", ".join(possible_categories)),
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="salesforce",
+        cli_config=SalesforceCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/sharepoint.py

@@ -0,0 +1,66 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+)
+from unstructured_ingest.connector.sharepoint import SimpleSharepointConfig
+
+
+@dataclass
+class SharepointCliConfig(SimpleSharepointConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                default=None,
+                type=str,
+                help="Sharepoint app client ID",
+            ),
+            click.Option(
+                ["--client-cred"],
+                default=None,
+                type=str,
+                help="Sharepoint app secret",
+            ),
+            click.Option(
+                ["--site"],
+                default=None,
+                type=str,
+                help="Sharepoint site url. Process either base url e.g \
+                    https://[tenant].sharepoint.com  or relative sites \
+                    https://[tenant].sharepoint.com/sites/<site_name>. \
+                    To process all sites within the tenant pass a site url as \
+                    https://[tenant]-admin.sharepoint.com.\
+                    This requires the app to be registered at a tenant level",
+            ),
+            click.Option(
+                ["--path"],
+                default="Shared Documents",
+                type=str,
+                help="Path from which to start parsing files. If the connector is to \
+                process all sites  within the tenant this filter will be applied to \
+                all sites document libraries. Default 'Shared Documents'",
+            ),
+            click.Option(
+                ["--files-only"],
+                is_flag=True,
+                default=False,
+                help="Process only files.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="sharepoint",
+        cli_config=SharepointCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/slack.py

@@ -0,0 +1,56 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.slack import SimpleSlackConfig
+
+
+@dataclass
+class SlackCliConfig(SimpleSlackConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--token"],
+                required=True,
+                type=str,
+                help="Bot token used to access Slack API, must have channels:history "
+                "scope for the bot user",
+            ),
+            click.Option(
+                ["--channels"],
+                required=True,
+                type=DelimitedString(),
+                help="Comma-delimited list of Slack channel IDs to pull messages from, "
+                "can be a public or private channel",
+            ),
+            click.Option(
+                ["--start-date"],
+                default=None,
+                type=str,
+                help="Start date/time in formats YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or "
+                "YYYY-MM-DD+HH:MM:SS or YYYY-MM-DDTHH:MM:SStz",
+            ),
+            click.Option(
+                ["--end-date"],
+                default=None,
+                type=str,
+                help="End date/time in formats YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS or "
+                "YYYY-MM-DD+HH:MM:SS or YYYY-MM-DDTHH:MM:SStz",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="slack",
+        cli_config=SlackCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/sql.py

@@ -0,0 +1,66 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.sql import SimpleSqlConfig
+from unstructured_ingest.interfaces import WriteConfig
+
+SQL_DRIVERS = {"postgresql", "sqlite"}
+
+
+@dataclass
+class SqlCliConfig(SimpleSqlConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--db-type"],
+                required=True,
+                type=click.Choice(SQL_DRIVERS),
+                help="Type of the database backend",
+            ),
+            click.Option(
+                ["--username"],
+                default=None,
+                type=str,
+                help="DB username",
+            ),
+            click.Option(
+                ["--password"],
+                default=None,
+                type=str,
+                help="DB password",
+            ),
+            click.Option(
+                ["--host"],
+                default=None,
+                type=str,
+                help="DB host",
+            ),
+            click.Option(
+                ["--port"],
+                default=None,
+                type=int,
+                help="DB host connection port",
+            ),
+            click.Option(
+                ["--database"],
+                default=None,
+                type=str,
+                help="Database name. For sqlite databases, this is the path to the .db file.",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="sql",
+        cli_config=SqlCliConfig,
+        write_config=WriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/vectara.py

@@ -0,0 +1,66 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.vectara import SimpleVectaraConfig, WriteConfig
+
+
+@dataclass
+class VectaraCliWriteConfig(SimpleVectaraConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--customer-id"],
+                required=True,
+                type=str,
+                help="The Vectara customer-id.",
+                envvar="VECTARA_CUSTOMER_ID",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--oauth-client-id"],
+                required=True,
+                type=str,
+                help="Vectara OAuth2 client ID.",
+                envvar="VECTARA_OAUTH_CLIENT_ID",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--oauth-secret"],
+                required=True,
+                type=str,
+                help="Vectara OAuth2 secret.",
+                envvar="VECTARA_OAUTH_SECRET",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--corpus-name"],
+                required=False,
+                type=str,
+                default=None,
+                help="The Vectara corpus-name.",
+            ),
+            click.Option(
+                ["--token-url"],
+                required=False,
+                default="https://vectara-prod-{}.auth.us-west-2.amazoncognito.com/oauth2/token",
+                type=str,
+                help="The Vectara endpoint for token refresh. Needs curly brackets for customer_id",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="vectara",
+        cli_config=VectaraCliWriteConfig,
+        additional_cli_options=[],
+        write_config=WriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/weaviate.py

@@ -0,0 +1,98 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.weaviate import SimpleWeaviateConfig, WeaviateWriteConfig
+
+CMD_NAME = "weaviate"
+
+
+@dataclass
+class WeaviateCliConfig(SimpleWeaviateConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--host-url"],
+                required=True,
+                help="Weaviate instance url",
+            ),
+            click.Option(
+                ["--class-name"],
+                default=None,
+                type=str,
+                help="Name of the class to push the records into, e.g: Pdf-elements",
+            ),
+            click.Option(
+                ["--access-token"], default=None, type=str, help="Used to create the bearer token."
+            ),
+            click.Option(
+                ["--refresh-token"],
+                default=None,
+                type=str,
+                help="Will tie this value to the bearer token. If not provided, "
+                "the authentication will expire once the lifetime of the access token is up.",
+            ),
+            click.Option(
+                ["--api-key"],
+                default=None,
+                type=str,
+            ),
+            click.Option(
+                ["--client-secret"],
+                default=None,
+                type=str,
+            ),
+            click.Option(
+                ["--scope"],
+                default=None,
+                type=DelimitedString(),
+            ),
+            click.Option(
+                ["--username"],
+                default=None,
+                type=str,
+            ),
+            click.Option(
+                ["--password"],
+                default=None,
+                type=str,
+            ),
+            click.Option(
+                ["--anonymous"],
+                is_flag=True,
+                default=False,
+                type=bool,
+                help="if set, all auth values will be ignored",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class WeaviateCliWriteConfig(WeaviateWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=int,
+                help="Number of records per batch",
+            )
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=WeaviateCliConfig,
+        additional_cli_options=[WeaviateCliWriteConfig],
+        write_config=WeaviateWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/wikipedia.py

@@ -0,0 +1,40 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.wikipedia import SimpleWikipediaConfig
+
+
+@dataclass
+class WikipediaCliConfig(SimpleWikipediaConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--page-title"],
+                required=True,
+                type=str,
+                help='Title of a Wikipedia page, e.g. "Open source software".',
+            ),
+            click.Option(
+                ["--auto-suggest"],
+                default=True,
+                is_flag=True,
+                help="Whether to automatically suggest a page if the exact page was not found."
+                " Set to False if the wrong Wikipedia page is fetched.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="wikipedia",
+        cli_config=WikipediaCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/common.py

@@ -0,0 +1,7 @@
+import logging
+
+from unstructured_ingest.logger import ingest_log_streaming_init
+
+
+def log_options(options: dict, verbose=False):
+    ingest_log_streaming_init(logging.DEBUG if verbose else logging.INFO)