unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/cli/cmds/github.py

@@ -1,54 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
-from unstructured_ingest.connector.github import SimpleGitHubConfig
-
-
-@dataclass
-class GithubCliConfig(SimpleGitHubConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--url"],
-                required=True,
-                type=str,
-                help="URL to GitHub repository, e.g. "
-                '"https://github.com/Unstructured-IO/unstructured", or '
-                'a repository owner/name pair, e.g. "Unstructured-IO/unstructured"',
-            ),
-            click.Option(
-                ["--git-access-token"],
-                default=None,
-                help="A GitHub or GitLab access token, "
-                "see https://docs.github.com/en/authentication or "
-                "https://docs.gitlab.com/ee/api/rest/index.html#personalprojectgroup-access-tokens",
-            ),
-            click.Option(
-                ["--git-branch"],
-                default=None,
-                type=str,
-                help="The branch for which to fetch files from. If not given,"
-                " the default repository branch is used.",
-            ),
-            click.Option(
-                ["--git-file-glob"],
-                default=None,
-                type=DelimitedString(),
-                help="A comma-separated list of file globs to limit which "
-                "types of files are accepted, e.g. '*.html,*.txt'",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="github",
-        cli_config=GithubCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/gitlab.py

@@ -1,54 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
-from unstructured_ingest.connector.gitlab import SimpleGitlabConfig
-
-
-@dataclass
-class GitlabCliConfig(SimpleGitlabConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--url"],
-                required=True,
-                type=str,
-                help="URL to GitHub repository, e.g. "
-                '"https://github.com/Unstructured-IO/unstructured", or '
-                'a repository owner/name pair, e.g. "Unstructured-IO/unstructured"',
-            ),
-            click.Option(
-                ["--git-access-token"],
-                default=None,
-                help="A GitHub or GitLab access token, "
-                "see https://docs.github.com/en/authentication or "
-                "https://docs.gitlab.com/ee/api/rest/index.html#personalprojectgroup-access-tokens",
-            ),
-            click.Option(
-                ["--git-branch"],
-                default=None,
-                type=str,
-                help="The branch for which to fetch files from. If not given,"
-                " the default repository branch is used.",
-            ),
-            click.Option(
-                ["--git-file-glob"],
-                default=None,
-                type=DelimitedString(),
-                help="A comma-separated list of file globs to limit which types of "
-                "files are accepted, e.g. '*.html,*.txt'",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="gitlab",
-        cli_config=GitlabCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/google_drive.py

@@ -1,49 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-    FileOrJson,
-)
-from unstructured_ingest.connector.google_drive import SimpleGoogleDriveConfig
-
-
-@dataclass
-class GoogleDriveCliConfig(SimpleGoogleDriveConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--drive-id"],
-                required=True,
-                type=str,
-                help="Google Drive File or Folder ID.",
-            ),
-            click.Option(
-                ["--service-account-key"],
-                required=True,
-                type=FileOrJson(),
-                help="Either the file path of the credentials file to use or a json string of "
-                "those values to use for authentication",
-            ),
-            click.Option(
-                ["--extension"],
-                default=None,
-                type=str,
-                help="Filters the files to be processed based on extension e.g. .jpg, .docx, etc.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="google-drive",
-        cli_config=GoogleDriveCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/hubspot.py

@@ -1,70 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliMixin, DelimitedString, Dict
-from unstructured_ingest.connector.hubspot import HubSpotObjectTypes, SimpleHubSpotConfig
-
-OBJECT_TYPES = {t.value for t in HubSpotObjectTypes}
-
-
-def validate_custom_property(ctx, param, value) -> t.Dict[str, t.List[str]]:
-    if not value:
-        return value
-    for k in value:
-        if k not in OBJECT_TYPES:
-            raise ValueError(f"Invalid object type: {k}, must be one of {OBJECT_TYPES}")
-        if not isinstance(value[k], list):
-            raise ValueError(f"Invalid type: {type(value[k])}, must be a Python list.")
-    return value
-
-
-@dataclass
-class HubSpotCliConfig(SimpleHubSpotConfig, CliMixin):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--api-token"],
-                required=True,
-                type=str,
-                help="Access token to perform operations on Hubspot. \
-                    Check \
-                    https://developers.hubspot.com/docs/api/private-apps/ \
-                    for more info",
-            ),
-            click.Option(
-                ["--object-types"],
-                default=None,
-                required=False,
-                type=DelimitedString(choices=OBJECT_TYPES),
-                is_flag=False,
-                help=f"Object to include in the process.\
-                    Must be a subset of {','.join(OBJECT_TYPES)}.\
-                    If the argument is omitted all objects listed will be processed.",
-            ),
-            click.Option(
-                ["--custom-properties"],
-                default=None,
-                required=False,
-                type=Dict(),
-                is_flag=False,
-                callback=validate_custom_property,
-                help="Custom property to process information from.\
-                    It should be a json-like string in the form\
-                        <object_type>:[<custom_property_id>, ..., <custom_property_id>]\
-                    Must be internal name of the variable. If the property is missing, \
-                        it will be omitted.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="hubspot",
-        cli_config=HubSpotCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/jira.py

@@ -1,71 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.jira import SimpleJiraConfig
-
-
-@dataclass
-class JiraCliConfig(SimpleJiraConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--api-token"],
-                required=True,
-                type=str,
-                help="API Token to authenticate into Jira (into Atlassian). \
-                    Check \
-                    https://developer.atlassian.com/cloud/jira/platform/basic-auth-for-rest-apis/ \
-                    for more info.",
-            ),
-            click.Option(
-                ["--url"],
-                required=True,
-                type=str,
-                help="URL to Atlassian (Jira) Cloud, e.g. "
-                '"unstructured-jira-connector-test.atlassian.net"',
-            ),
-            click.Option(
-                ["--user-email"],
-                required=True,
-                type=str,
-                help="Email to authenticate into Atlassian (Jira) Cloud.",
-            ),
-            click.Option(
-                ["--projects"],
-                default=None,
-                type=DelimitedString(),
-                help="Comma-delimited Project ids or keys. Use Jira UI or the "
-                "API to find or obtain keys. Alternatively, use API to obtain ids.",
-            ),
-            click.Option(
-                ["--boards"],
-                default=None,
-                type=DelimitedString(),
-                help="Comma-delimited Board ids. Check board URL, or use the "
-                "API to find the board ids.",
-            ),
-            click.Option(
-                ["--issues"],
-                default=None,
-                type=DelimitedString(),
-                help="Comma-delimited Issue ids or keys. Use Jira UI or the API to "
-                "find or obtain keys. Alternatively, use API to obtain ids.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="jira",
-        cli_config=JiraCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/kafka.py

@@ -1,102 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliConfig
-from unstructured_ingest.connector.kafka import KafkaWriteConfig, SimpleKafkaConfig
-
-CMD_NAME = "kafka"
-
-
-@dataclass
-class KafkaCliConfig(SimpleKafkaConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--bootstrap-server"], required=True, type=str, help="Broker server hostname"
-            ),
-            click.Option(
-                ["--port"],
-                required=True,
-                type=str,
-                help="The bootstrap port",
-            ),
-            click.Option(
-                ["--topic"],
-                required=True,
-                type=str,
-                help="The topic to write into.'",
-            ),
-            click.Option(
-                ["--kafka-api-key"],
-                required=False,
-                type=str,
-                help="The API KEY",
-            ),
-            click.Option(
-                ["--secret"],
-                required=False,
-                type=str,
-                help="The secret",
-            ),
-            click.Option(
-                ["--num-messages-to-consume"],
-                required=False,
-                type=int,
-                default=1,
-                help="The number of messages to consume before unblocking the consumer",
-            ),
-            click.Option(
-                ["--timeout"],
-                required=False,
-                type=float,
-                default=1.0,
-                help="Maximum time to block waiting for message(Seconds)",
-            ),
-            click.Option(
-                ["--confluent"],
-                required=False,
-                type=bool,
-                default=True,
-                help="Whether this Kafka instance is from Confluent",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class KafkaCliWriteConfig(KafkaWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=4,
-                type=int,
-                help="Number of records per batch",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=KafkaCliConfig,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=KafkaCliConfig,
-        additional_cli_options=[KafkaCliWriteConfig],
-        write_config=KafkaWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/local.py

@@ -1,43 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.local import SimpleLocalConfig
-
-
-@dataclass
-class LocalCliConfig(SimpleLocalConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--input-path"],
-                required=True,
-                type=click.Path(file_okay=True, dir_okay=True, exists=True),
-                help="Path to the location in the local file system that will be processed.",
-            ),
-            click.Option(
-                ["--file-glob"],
-                default=None,
-                type=DelimitedString(),
-                help="A comma-separated list of file globs to limit which types of "
-                "local files are accepted, e.g. '*.html,*.txt'",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="local",
-        cli_config=LocalCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/mongodb.py

@@ -1,72 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
-from unstructured_ingest.connector.mongodb import SimpleMongoDBConfig
-from unstructured_ingest.interfaces import WriteConfig
-
-CMD_NAME = "mongodb"
-
-
-@dataclass
-class MongoDBCliConfig(SimpleMongoDBConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--uri"],
-                help="URI to user when connecting",
-            ),
-            click.Option(
-                ["--host"],
-                type=DelimitedString(),
-                help="hostname or IP address or Unix domain socket path of a single mongod or "
-                "mongos instance to connect to, or a list of hostnames",
-            ),
-            click.Option(["--port"], type=int, default=27017),
-            click.Option(
-                ["--database"], type=str, required=True, help="database name to connect to"
-            ),
-            click.Option(
-                ["--collection"], required=True, type=str, help="collection name to connect to"
-            ),
-        ]
-        return options
-
-
-@dataclass
-class MongoDBReadConfig(SimpleMongoDBConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=click.IntRange(0),
-                help="how many records to read at a time per process",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name=CMD_NAME,
-        cli_config=MongoDBCliConfig,
-        additional_cli_options=[MongoDBReadConfig],
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=MongoDBCliConfig,
-        write_config=WriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/notion.py

@@ -1,48 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.notion.connector import SimpleNotionConfig
-
-
-@dataclass
-class NotionCliConfig(SimpleNotionConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--notion-api-key"],
-                required=True,
-                type=str,
-                help="API key for Notion api",
-            ),
-            click.Option(
-                ["--page-ids"],
-                default=None,
-                type=DelimitedString(),
-                help="Notion page IDs to pull text from",
-            ),
-            click.Option(
-                ["--database-ids"],
-                default=None,
-                type=DelimitedString(),
-                help="Notion database IDs to pull text from",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="notion",
-        cli_config=NotionCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/onedrive.py

@@ -1,66 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    CliRecursiveConfig,
-)
-from unstructured_ingest.connector.onedrive import SimpleOneDriveConfig
-
-
-@dataclass
-class OnedriveCliConfig(SimpleOneDriveConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                required=True,
-                type=str,
-                help="Microsoft app client ID",
-            ),
-            click.Option(
-                ["--client-cred"],
-                required=True,
-                type=str,
-                help="Microsoft App client secret",
-            ),
-            click.Option(
-                ["--user-pname"],
-                required=True,
-                type=str,
-                help="User principal name, usually is your Azure AD email.",
-            ),
-            click.Option(
-                ["--tenant"],
-                default="common",
-                type=str,
-                help="ID or domain name associated with your Azure AD instance",
-            ),
-            click.Option(
-                ["--path"],
-                default=None,
-                type=str,
-                help="Folder to start parsing files from.",
-            ),
-            click.Option(
-                ["--authority-url"],
-                default="https://login.microsoftonline.com",
-                type=str,
-                help="Authentication token provider for Microsoft apps, default is "
-                "https://login.microsoftonline.com",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="onedrive",
-        cli_config=OnedriveCliConfig,
-        additional_cli_options=[CliRecursiveConfig],
-    )
-    return cmd_cls