unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/cli/cmds/local.py

@@ -0,0 +1,43 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.local import SimpleLocalConfig
+
+
+@dataclass
+class LocalCliConfig(SimpleLocalConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--input-path"],
+                required=True,
+                type=click.Path(file_okay=True, dir_okay=True, exists=True),
+                help="Path to the location in the local file system that will be processed.",
+            ),
+            click.Option(
+                ["--file-glob"],
+                default=None,
+                type=DelimitedString(),
+                help="A comma-separated list of file globs to limit which types of "
+                "local files are accepted, e.g. '*.html,*.txt'",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="local",
+        cli_config=LocalCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/mongodb.py

@@ -0,0 +1,72 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.mongodb import SimpleMongoDBConfig
+from unstructured_ingest.interfaces import WriteConfig
+
+CMD_NAME = "mongodb"
+
+
+@dataclass
+class MongoDBCliConfig(SimpleMongoDBConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--uri"],
+                help="URI to user when connecting",
+            ),
+            click.Option(
+                ["--host"],
+                type=DelimitedString(),
+                help="hostname or IP address or Unix domain socket path of a single mongod or "
+                "mongos instance to connect to, or a list of hostnames",
+            ),
+            click.Option(["--port"], type=int, default=27017),
+            click.Option(
+                ["--database"], type=str, required=True, help="database name to connect to"
+            ),
+            click.Option(
+                ["--collection"], required=True, type=str, help="collection name to connect to"
+            ),
+        ]
+        return options
+
+
+@dataclass
+class MongoDBReadConfig(SimpleMongoDBConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=click.IntRange(0),
+                help="how many records to read at a time per process",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=MongoDBCliConfig,
+        additional_cli_options=[MongoDBReadConfig],
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=MongoDBCliConfig,
+        write_config=WriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/notion.py

@@ -0,0 +1,48 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.notion.connector import SimpleNotionConfig
+
+
+@dataclass
+class NotionCliConfig(SimpleNotionConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--notion-api-key"],
+                required=True,
+                type=str,
+                help="API key for Notion api",
+            ),
+            click.Option(
+                ["--page-ids"],
+                default=None,
+                type=DelimitedString(),
+                help="Notion page IDs to pull text from",
+            ),
+            click.Option(
+                ["--database-ids"],
+                default=None,
+                type=DelimitedString(),
+                help="Notion database IDs to pull text from",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="notion",
+        cli_config=NotionCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/onedrive.py

@@ -0,0 +1,66 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+)
+from unstructured_ingest.connector.onedrive import SimpleOneDriveConfig
+
+
+@dataclass
+class OnedriveCliConfig(SimpleOneDriveConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                required=True,
+                type=str,
+                help="Microsoft app client ID",
+            ),
+            click.Option(
+                ["--client-cred"],
+                required=True,
+                type=str,
+                help="Microsoft App client secret",
+            ),
+            click.Option(
+                ["--user-pname"],
+                required=True,
+                type=str,
+                help="User principal name, usually is your Azure AD email.",
+            ),
+            click.Option(
+                ["--tenant"],
+                default="common",
+                type=str,
+                help="ID or domain name associated with your Azure AD instance",
+            ),
+            click.Option(
+                ["--path"],
+                default=None,
+                type=str,
+                help="Folder to start parsing files from.",
+            ),
+            click.Option(
+                ["--authority-url"],
+                default="https://login.microsoftonline.com",
+                type=str,
+                help="Authentication token provider for Microsoft apps, default is "
+                "https://login.microsoftonline.com",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="onedrive",
+        cli_config=OnedriveCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/opensearch.py

@@ -0,0 +1,117 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.cmds.elasticsearch import ElasticsearchCliWriteConfig
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.opensearch import SimpleOpenSearchConfig
+
+CMD_NAME = "opensearch"
+
+
+@dataclass
+class OpenSearchCliConfig(SimpleOpenSearchConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--index-name"],
+                required=True,
+                type=str,
+                help="Name of the OpenSearch index to pull data from, or upload data to.",
+            ),
+            click.Option(
+                ["--hosts"],
+                type=DelimitedString(),
+                help='List of the OpenSearch hosts to connect to, e.g. "http://localhost:9200"',
+            ),
+            click.Option(
+                ["--fields"],
+                type=DelimitedString(),
+                default=[],
+                help="If provided, will limit the fields returned by OpenSearch "
+                "to this comma-delimited list",
+            ),
+            click.Option(
+                ["--username"], type=str, default=None, help="username when using basic auth"
+            ),
+            click.Option(
+                ["--password"],
+                type=str,
+                default=None,
+                help="password when using basic auth",
+            ),
+            click.Option(
+                ["--use-ssl"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="use ssl for the connection",
+            ),
+            click.Option(
+                ["--verify-certs"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="whether to verify SSL certificates",
+            ),
+            click.Option(
+                ["--ssl-show-warn"],
+                type=bool,
+                default=False,
+                is_flag=True,
+                help="show warning when verify certs is disabled",
+            ),
+            click.Option(
+                ["--ca-certs"],
+                type=click.Path(),
+                default=None,
+                help="path to CA bundle",
+            ),
+            click.Option(
+                ["--client-cert"],
+                type=click.Path(),
+                default=None,
+                help="path to the file containing the private key and the certificate,"
+                " or cert only if using client_key",
+            ),
+            click.Option(
+                ["--client-key"],
+                type=click.Path(),
+                default=None,
+                help="path to the file containing the private key"
+                " if using separate cert and key files",
+            ),
+            click.Option(
+                ["--batch-size"],
+                default=100,
+                type=click.IntRange(0),
+                help="how many records to read at a time per process",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="opensearch",
+        cli_config=OpenSearchCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="opensearch",
+        cli_config=OpenSearchCliConfig,
+        additional_cli_options=[ElasticsearchCliWriteConfig],
+        addition_configs={
+            "connector_config": SimpleOpenSearchConfig,
+            "write_config": ElasticsearchCliWriteConfig,
+        },
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/outlook.py

@@ -0,0 +1,67 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.outlook import SimpleOutlookConfig
+
+
+@dataclass
+class OutlookCliConfig(SimpleOutlookConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                required=True,
+                type=str,
+                help="Microsoft app client ID",
+            ),
+            click.Option(
+                ["--user-email"],
+                required=True,
+                type=str,
+                help="Outlook email to download messages from.",
+            ),
+            click.Option(
+                ["--tenant"],
+                default="common",
+                help="ID or domain name associated with your Azure AD instance",
+            ),
+            click.Option(
+                ["--outlook-folders"],
+                default=None,
+                type=DelimitedString(),
+                help="Folders to download email messages from. "
+                "Do not specify subfolders. Use quotes if spaces in folder names.",
+            ),
+            click.Option(
+                ["--client-cred"],
+                default=None,
+                type=str,
+                help="Microsoft App client secret",
+            ),
+            click.Option(
+                ["--authority-url"],
+                default="https://login.microsoftonline.com",
+                type=str,
+                help="Authentication token provider for Microsoft apps, default is "
+                "https://login.microsoftonline.com",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="outlook",
+        cli_config=OutlookCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/pinecone.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.pinecone import PineconeWriteConfig, SimplePineconeConfig
+
+
+@dataclass
+class PineconeCliConfig(SimplePineconeConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-key"],
+                required=True,
+                type=str,
+                help="API key used for authenticating to a Pinecone instance.",
+                envvar="PINECONE_API_KEY",
+                show_envvar=True,
+            ),
+            click.Option(
+                ["--index-name"],
+                required=True,
+                type=str,
+                help="The name of the pinecone index to connect to.",
+            ),
+            click.Option(
+                ["--environment"],
+                required=True,
+                type=str,
+                help="The environment where the index lives. Eg. 'gcp-starter' or 'us-east1-gcp'",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class PineconeCliWriteConfig(PineconeWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=50,
+                type=int,
+                help="Number of records per batch",
+            ),
+            click.Option(
+                ["--num-processes"],
+                default=2,
+                type=int,
+                help="Number of parallel processes with which to upload elements",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="pinecone",
+        cli_config=PineconeCliConfig,
+        additional_cli_options=[PineconeCliWriteConfig],
+        write_config=PineconeWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/qdrant.py

@@ -0,0 +1,124 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.qdrant import QdrantWriteConfig, SimpleQdrantConfig
+
+
+@dataclass
+class QdrantCliConfig(SimpleQdrantConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--collection-name"],
+                required=True,
+                type=str,
+                help="The name of the Qdrant collection to use.",
+            ),
+            click.Option(
+                ["--location"],
+                type=str,
+                help="The location of the Qdrant cluster.",
+            ),
+            click.Option(
+                ["--url"],
+                type=str,
+                help="The location of the Qdrant cluster.",
+            ),
+            click.Option(
+                ["--port"],
+                type=int,
+                default=6333,
+                help="Port of the REST API interface. Default: 6333.",
+            ),
+            click.Option(
+                ["--grpc-port"],
+                type=int,
+                default=6334,
+                help="Port of the gRPC interface. Default: 6334.",
+            ),
+            click.Option(
+                ["--prefer-grpc"],
+                type=bool,
+                is_flag=True,
+                help="Whether to use gPRC interface whenever possible in methods. Default: False.",
+            ),
+            click.Option(
+                ["--https"],
+                type=bool,
+                is_flag=True,
+                help="Whether to use HTTPS(SSL) protocol. Default: False.",
+            ),
+            click.Option(
+                ["--prefix"],
+                type=str,
+                help="Prefix to add the REST API endpoints.",
+            ),
+            click.Option(
+                ["--timeout"],
+                type=int,
+                help="Timeout for operations. Default: 5.0 seconds for REST, unlimited for gRPC.",
+            ),
+            click.Option(
+                ["--host"],
+                type=str,
+                help="Host name of the Qdrant service.",
+            ),
+            click.Option(
+                ["--path"],
+                type=str,
+                help="Persistence path for QdrantLocal.",
+            ),
+            click.Option(
+                ["--force-disable-check-same-thread"],
+                type=bool,
+                is_flag=True,
+                help="Whether to force disable check same thread for QdrantLocal.",
+            ),
+            click.Option(
+                ["--api-key"],
+                type=str,
+                help="API key for authentication in Qdrant Cloud. Default: None.",
+                envvar="QDRANT_API_KEY",
+                show_envvar=True,
+            ),
+        ]
+        return options
+
+
+@dataclass
+class QdrantCliWriteConfig(QdrantWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=50,
+                type=int,
+                help="Number of points to upload per batch",
+            ),
+            click.Option(
+                ["--num-processes"],
+                default=2,
+                type=int,
+                help="Number of parallel processes with which to upload",
+            ),
+        ]
+        return options
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name="qdrant",
+        cli_config=QdrantCliConfig,
+        additional_cli_options=[QdrantCliWriteConfig],
+        write_config=QdrantWriteConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/reddit.py

@@ -0,0 +1,67 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.reddit import SimpleRedditConfig
+
+
+@dataclass
+class RedditCliConfig(SimpleRedditConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--client-id"],
+                required=True,
+                type=str,
+                help="The client ID, see "
+                "https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#prerequisites"  # noqa: E501
+                " for more information.",
+            ),
+            click.Option(
+                ["--client-secret"],
+                required=True,
+                type=str,
+                help="The client secret, see "
+                "https://praw.readthedocs.io/en/stable/getting_started/quick_start.html#prerequisites"  # noqa: E501
+                " for more information.",
+            ),
+            click.Option(
+                ["--subreddit-name"],
+                required=True,
+                type=str,
+                help='The name of a subreddit, without the "r\\", e.g. "machinelearning"',
+            ),
+            click.Option(
+                ["--search-query"],
+                default=None,
+                type=str,
+                help="If set, return posts using this query. Otherwise, use hot posts.",
+            ),
+            click.Option(
+                ["--num-posts"],
+                required=True,
+                type=click.IntRange(0),
+                help="If set, limits the number of posts to pull in.",
+            ),
+            click.Option(
+                ["--user-agent"],
+                required=True,
+                type=str,
+                help="user agent request header to use when calling Reddit API",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="reddit",
+        cli_config=RedditCliConfig,
+    )
+    return cmd_cls