unstructured-ingest 0.0.0__py3-none-any.whl

unstructured_ingest/cli/cmds/jira.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.jira import SimpleJiraConfig
+
+
+@dataclass
+class JiraCliConfig(SimpleJiraConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-token"],
+                required=True,
+                type=str,
+                help="API Token to authenticate into Jira (into Atlassian). \
+                    Check \
+                    https://developer.atlassian.com/cloud/jira/platform/basic-auth-for-rest-apis/ \
+                    for more info.",
+            ),
+            click.Option(
+                ["--url"],
+                required=True,
+                type=str,
+                help="URL to Atlassian (Jira) Cloud, e.g. "
+                '"unstructured-jira-connector-test.atlassian.net"',
+            ),
+            click.Option(
+                ["--user-email"],
+                required=True,
+                type=str,
+                help="Email to authenticate into Atlassian (Jira) Cloud.",
+            ),
+            click.Option(
+                ["--projects"],
+                default=None,
+                type=DelimitedString(),
+                help="Comma-delimited Project ids or keys. Use Jira UI or the "
+                "API to find or obtain keys. Alternatively, use API to obtain ids.",
+            ),
+            click.Option(
+                ["--boards"],
+                default=None,
+                type=DelimitedString(),
+                help="Comma-delimited Board ids. Check board URL, or use the "
+                "API to find the board ids.",
+            ),
+            click.Option(
+                ["--issues"],
+                default=None,
+                type=DelimitedString(),
+                help="Comma-delimited Issue ids or keys. Use Jira UI or the API to "
+                "find or obtain keys. Alternatively, use API to obtain ids.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="jira",
+        cli_config=JiraCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/kafka.py

@@ -0,0 +1,102 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.kafka import KafkaWriteConfig, SimpleKafkaConfig
+
+CMD_NAME = "kafka"
+
+
+@dataclass
+class KafkaCliConfig(SimpleKafkaConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--bootstrap-server"], required=True, type=str, help="Broker server hostname"
+            ),
+            click.Option(
+                ["--port"],
+                required=True,
+                type=str,
+                help="The bootstrap port",
+            ),
+            click.Option(
+                ["--topic"],
+                required=True,
+                type=str,
+                help="The topic to write into.'",
+            ),
+            click.Option(
+                ["--kafka-api-key"],
+                required=False,
+                type=str,
+                help="The API KEY",
+            ),
+            click.Option(
+                ["--secret"],
+                required=False,
+                type=str,
+                help="The secret",
+            ),
+            click.Option(
+                ["--num-messages-to-consume"],
+                required=False,
+                type=int,
+                default=1,
+                help="The number of messages to consume before unblocking the consumer",
+            ),
+            click.Option(
+                ["--timeout"],
+                required=False,
+                type=float,
+                default=1.0,
+                help="Maximum time to block waiting for message(Seconds)",
+            ),
+            click.Option(
+                ["--confluent"],
+                required=False,
+                type=bool,
+                default=True,
+                help="Whether this Kafka instance is from Confluent",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class KafkaCliWriteConfig(KafkaWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=4,
+                type=int,
+                help="Number of records per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=KafkaCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=KafkaCliConfig,
+        additional_cli_options=[KafkaCliWriteConfig],
+        write_config=KafkaWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/local.py

@@ -0,0 +1,43 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.local import SimpleLocalConfig
+
+
+@dataclass
+class LocalCliConfig(SimpleLocalConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--input-path"],
+                required=True,
+                type=click.Path(file_okay=True, dir_okay=True, exists=True),
+                help="Path to the location in the local file system that will be processed.",
+            ),
+            click.Option(
+                ["--file-glob"],
+                default=None,
+                type=DelimitedString(),
+                help="A comma-separated list of file globs to limit which types of "
+                "local files are accepted, e.g. '*.html,*.txt'",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="local",
+        cli_config=LocalCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/mongodb.py

@@ -0,0 +1,72 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.mongodb import SimpleMongoDBConfig
+from unstructured_ingest.interfaces import WriteConfig
+
+CMD_NAME = "mongodb"
+
+
+@dataclass
+class MongoDBCliConfig(SimpleMongoDBConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--uri"],
+                help="URI to user when connecting",
+            ),
+            click.Option(
+                ["--host"],
+                type=DelimitedString(),
+                help="hostname or IP address or Unix domain socket path of a single mongod or "
+                "mongos instance to connect to, or a list of hostnames",
+            ),
+            click.Option(["--port"], type=int, default=27017),
+            click.Option(
+                ["--database"], type=str, required=True, help="database name to connect to"
+            ),
+            click.Option(
+                ["--collection"], required=True, type=str, help="collection name to connect to"
+            ),
+        ]
+        return options
+
+
+@dataclass
+class MongoDBReadConfig(SimpleMongoDBConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=click.IntRange(0),
+                help="how many records to read at a time per process",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=MongoDBCliConfig,
+        additional_cli_options=[MongoDBReadConfig],
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=MongoDBCliConfig,
+        write_config=WriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/notion.py

@@ -0,0 +1,48 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.notion.connector import SimpleNotionConfig
+
+
+@dataclass
+class NotionCliConfig(SimpleNotionConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--notion-api-key"],
+                required=True,
+                type=str,
+                help="API key for Notion api",
+            ),
+            click.Option(
+                ["--page-ids"],
+                default=None,
+                type=DelimitedString(),
+                help="Notion page IDs to pull text from",
+            ),
+            click.Option(
+                ["--database-ids"],
+                default=None,
+                type=DelimitedString(),
+                help="Notion database IDs to pull text from",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="notion",
+        cli_config=NotionCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/onedrive.py

@@ -0,0 +1,66 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+)
+from unstructured_ingest.connector.onedrive import SimpleOneDriveConfig
+
+
+@dataclass
+class OnedriveCliConfig(SimpleOneDriveConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                required=True,
+                type=str,
+                help="Microsoft app client ID",
+            ),
+            click.Option(
+                ["--client-cred"],
+                required=True,
+                type=str,
+                help="Microsoft App client secret",
+            ),
+            click.Option(
+                ["--user-pname"],
+                required=True,
+                type=str,
+                help="User principal name, usually is your Azure AD email.",
+            ),
+            click.Option(
+                ["--tenant"],
+                default="common",
+                type=str,
+                help="ID or domain name associated with your Azure AD instance",
+            ),
+            click.Option(
+                ["--path"],
+                default=None,
+                type=str,
+                help="Folder to start parsing files from.",
+            ),
+            click.Option(
+                ["--authority-url"],
+                default="https://login.microsoftonline.com",
+                type=str,
+                help="Authentication token provider for Microsoft apps, default is "
+                "https://login.microsoftonline.com",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="onedrive",
+        cli_config=OnedriveCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/opensearch.py

@@ -0,0 +1,117 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.cmds.elasticsearch import ElasticsearchCliWriteConfig
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.opensearch import SimpleOpenSearchConfig
+
+CMD_NAME = "opensearch"
+
+
+@dataclass
+class OpenSearchCliConfig(SimpleOpenSearchConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--index-name"],
+                required=True,
+                type=str,
+                help="Name of the OpenSearch index to pull data from, or upload data to.",
+            ),
+            click.Option(
+                ["--hosts"],
+                type=DelimitedString(),
+                help='List of the OpenSearch hosts to connect to, e.g. "http://localhost:9200"',
+            ),
+            click.Option(
+                ["--fields"],
+                type=DelimitedString(),
+                default=[],
+                help="If provided, will limit the fields returned by OpenSearch "
+                "to this comma-delimited list",
+            ),
+            click.Option(
+                ["--username"], type=str, default=None, help="username when using basic auth"
+            ),
+            click.Option(
+                ["--password"],
+                type=str,
+                default=None,
+                help="password when using basic auth",
+            ),
+            click.Option(
+                ["--use-ssl"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="use ssl for the connection",
+            ),
+            click.Option(
+                ["--verify-certs"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="whether to verify SSL certificates",
+            ),
+            click.Option(
+                ["--ssl-show-warn"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="show warning when verify certs is disabled",
+            ),
+            click.Option(
+                ["--ca-certs"],
+                type=click.Path(),
+                default=None,
+                help="path to CA bundle",
+            ),
+            click.Option(
+                ["--client-cert"],
+                type=click.Path(),
+                default=None,
+                help="path to the file containing the private key and the certificate,"
+                " or cert only if using client_key",
+            ),
+            click.Option(
+                ["--client-key"],
+                type=click.Path(),
+                default=None,
+                help="path to the file containing the private key"
+                " if using separate cert and key files",
+            ),
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=click.IntRange(0),
+                help="how many records to read at a time per process",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="opensearch",
+        cli_config=OpenSearchCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="opensearch",
+        cli_config=OpenSearchCliConfig,
+        additional_cli_options=[ElasticsearchCliWriteConfig],
+        addition_configs={
+            "connector_config": SimpleOpenSearchConfig,
+            "write_config": ElasticsearchCliWriteConfig,
+        },
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/outlook.py

@@ -0,0 +1,67 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.outlook import SimpleOutlookConfig
+
+
+@dataclass
+class OutlookCliConfig(SimpleOutlookConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                required=True,
+                type=str,
+                help="Microsoft app client ID",
+            ),
+            click.Option(
+                ["--user-email"],
+                required=True,
+                type=str,
+                help="Outlook email to download messages from.",
+            ),
+            click.Option(
+                ["--tenant"],
+                default="common",
+                help="ID or domain name associated with your Azure AD instance",
+            ),
+            click.Option(
+                ["--outlook-folders"],
+                default=None,
+                type=DelimitedString(),
+                help="Folders to download email messages from. "
+                "Do not specify subfolders. Use quotes if spaces in folder names.",
+            ),
+            click.Option(
+                ["--client-cred"],
+                default=None,
+                type=str,
+                help="Microsoft App client secret",
+            ),
+            click.Option(
+                ["--authority-url"],
+                default="https://login.microsoftonline.com",
+                type=str,
+                help="Authentication token provider for Microsoft apps, default is "
+                "https://login.microsoftonline.com",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="outlook",
+        cli_config=OutlookCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/pinecone.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.pinecone import PineconeWriteConfig, SimplePineconeConfig
+
+
+@dataclass
+class PineconeCliConfig(SimplePineconeConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-key"],
+                required=True,
+                type=str,
+                help="API key used for authenticating to a Pinecone instance.",
+                envvar="PINECONE_API_KEY",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--index-name"],
+                required=True,
+                type=str,
+                help="The name of the pinecone index to connect to.",
+            ),
+            click.Option(
+                ["--environment"],
+                required=True,
+                type=str,
+                help="The environment where the index lives. Eg. 'gcp-starter' or 'us-east1-gcp'",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class PineconeCliWriteConfig(PineconeWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=50,
+                type=int,
+                help="Number of records per batch",
+            ),
+            click.Option(
+                ["--num-processes"],
+                default=2,
+                type=int,
+                help="Number of parallel processes with which to upload elements",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="pinecone",
+        cli_config=PineconeCliConfig,
+        additional_cli_options=[PineconeCliWriteConfig],
+        write_config=PineconeWriteConfig,
+    )
+    return cmd_cls