unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/cli/cmds/fsspec/dropbox.py

@@ -0,0 +1,51 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.fsspec.dropbox import (
+    DropboxWriteConfig,
+    SimpleDropboxConfig,
+)
+
+CMD_NAME = "dropbox"
+
+
+@dataclass
+class DropboxCliConfig(SimpleDropboxConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--token"],
+                required=True,
+                type=str,
+                help="Dropbox access token.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=DropboxCliConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=DropboxCliConfig,
+        write_config=DropboxWriteConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/fsspec.py

@@ -0,0 +1,15 @@
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+
+CMD_NAME = "fsspec"
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(cmd_name=CMD_NAME, is_fsspec=True)
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(cmd_name=CMD_NAME, is_fsspec=True)
+    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/gcs.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    FileOrJson,
+)
+from unstructured_ingest.connector.fsspec.gcs import GcsWriteConfig, SimpleGcsConfig
+
+CMD_NAME = "gcs"
+
+
+@dataclass
+class GcsCliConfig(SimpleGcsConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        help_string = """
+        Options:
+        - ``None``, GCSFS will attempt to guess your credentials in the
+      following order: gcloud CLI default, gcsfs cached token, google compute
+      metadata service, anonymous.
+        - ``'google_default'``, your default gcloud credentials will be used,
+          which are typically established by doing ``gcloud login`` in a terminal.
+        - ``'cache'``, credentials from previously successful gcsfs
+          authentication will be used (use this after "browser" auth succeeded)
+        - ``'anon'``, no authentication is performed, and you can only
+          access data which is accessible to allUsers (in this case, the project and
+          access level parameters are meaningless)
+        - ``'browser'``, you get an access code with which you can
+          authenticate via a specially provided URL
+        - if ``'cloud'``, we assume we are running within google compute
+          or google container engine, and query the internal metadata directly for
+          a token.
+        - you may supply a token generated by the
+          [gcloud](https://cloud.google.com/sdk/docs/)
+          utility; this is either a python dictionary or the name of a file
+          containing the JSON returned by logging in with the gcloud CLI tool.
+        """
+        options = [
+            click.Option(
+                ["--service-account-key"],
+                default=None,
+                type=FileOrJson(allow_raw_str=True),
+                help=help_string,
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=GcsCliConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=GcsCliConfig,
+        write_config=GcsWriteConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/s3.py

@@ -0,0 +1,74 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.fsspec.s3 import S3WriteConfig, SimpleS3Config
+
+CMD_NAME = "s3"
+
+
+@dataclass
+class S3CliConfig(SimpleS3Config, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--anonymous"],
+                is_flag=True,
+                default=False,
+                help="Connect to s3 without local AWS credentials.",
+            ),
+            click.Option(
+                ["--endpoint-url"],
+                type=str,
+                default=None,
+                help="Use this endpoint_url, if specified. Needed for "
+                "connecting to non-AWS S3 buckets.",
+            ),
+            click.Option(
+                ["--key"],
+                type=str,
+                default=None,
+                help="If not anonymous, use this access key ID, if specified. Takes precedence "
+                "over `aws_access_key_id` in client_kwargs.",
+            ),
+            click.Option(
+                ["--secret"],
+                type=str,
+                default=None,
+                help="If not anonymous, use this secret access key, if specified.",
+            ),
+            click.Option(
+                ["--token"],
+                type=str,
+                default=None,
+                help="If not anonymous, use this security token, if specified.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd():
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=S3CliConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=S3CliConfig,
+        write_config=S3WriteConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/fsspec/sftp.py

@@ -0,0 +1,58 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+)
+from unstructured_ingest.connector.fsspec.sftp import SimpleSftpConfig
+
+CMD_NAME = "sftp"
+
+
+@dataclass
+class SftpCliConfig(SimpleSftpConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--username"],
+                required=True,
+                type=str,
+                help="Username for sftp connection",
+            ),
+            click.Option(
+                ["--password"],
+                required=True,
+                type=str,
+                help="Password for sftp connection",
+            ),
+            click.Option(
+                ["--look-for-keys"],
+                required=False,
+                default=False,
+                is_flag=True,
+                type=bool,
+                help="Whether to search for private key files in ~/.ssh/",
+            ),
+            click.Option(
+                ["--allow-agent"],
+                required=False,
+                default=False,
+                is_flag=True,
+                type=bool,
+                help="Whether to connect to the SSH agent.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=SftpCliConfig,
+        is_fsspec=True,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/github.py

@@ -0,0 +1,54 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.github import SimpleGitHubConfig
+
+
+@dataclass
+class GithubCliConfig(SimpleGitHubConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--url"],
+                required=True,
+                type=str,
+                help="URL to GitHub repository, e.g. "
+                '"https://github.com/Unstructured-IO/unstructured", or '
+                'a repository owner/name pair, e.g. "Unstructured-IO/unstructured"',
+            ),
+            click.Option(
+                ["--git-access-token"],
+                default=None,
+                help="A GitHub or GitLab access token, "
+                "see https://docs.github.com/en/authentication or "
+                "https://docs.gitlab.com/ee/api/rest/index.html#personalprojectgroup-access-tokens",
+            ),
+            click.Option(
+                ["--git-branch"],
+                default=None,
+                type=str,
+                help="The branch for which to fetch files from. If not given,"
+                " the default repository branch is used.",
+            ),
+            click.Option(
+                ["--git-file-glob"],
+                default=None,
+                type=DelimitedString(),
+                help="A comma-separated list of file globs to limit which "
+                "types of files are accepted, e.g. '*.html,*.txt'",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="github",
+        cli_config=GithubCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/gitlab.py

@@ -0,0 +1,54 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
+from unstructured_ingest.connector.gitlab import SimpleGitlabConfig
+
+
+@dataclass
+class GitlabCliConfig(SimpleGitlabConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--url"],
+                required=True,
+                type=str,
+                help="URL to GitHub repository, e.g. "
+                '"https://github.com/Unstructured-IO/unstructured", or '
+                'a repository owner/name pair, e.g. "Unstructured-IO/unstructured"',
+            ),
+            click.Option(
+                ["--git-access-token"],
+                default=None,
+                help="A GitHub or GitLab access token, "
+                "see https://docs.github.com/en/authentication or "
+                "https://docs.gitlab.com/ee/api/rest/index.html#personalprojectgroup-access-tokens",
+            ),
+            click.Option(
+                ["--git-branch"],
+                default=None,
+                type=str,
+                help="The branch for which to fetch files from. If not given,"
+                " the default repository branch is used.",
+            ),
+            click.Option(
+                ["--git-file-glob"],
+                default=None,
+                type=DelimitedString(),
+                help="A comma-separated list of file globs to limit which types of "
+                "files are accepted, e.g. '*.html,*.txt'",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="gitlab",
+        cli_config=GitlabCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/google_drive.py

@@ -0,0 +1,49 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    CliRecursiveConfig,
+    FileOrJson,
+)
+from unstructured_ingest.connector.google_drive import SimpleGoogleDriveConfig
+
+
+@dataclass
+class GoogleDriveCliConfig(SimpleGoogleDriveConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--drive-id"],
+                required=True,
+                type=str,
+                help="Google Drive File or Folder ID.",
+            ),
+            click.Option(
+                ["--service-account-key"],
+                required=True,
+                type=FileOrJson(),
+                help="Either the file path of the credentials file to use or a json string of "
+                "those values to use for authentication",
+            ),
+            click.Option(
+                ["--extension"],
+                default=None,
+                type=str,
+                help="Filters the files to be processed based on extension e.g. .jpg, .docx, etc.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="google-drive",
+        cli_config=GoogleDriveCliConfig,
+        additional_cli_options=[CliRecursiveConfig],
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/hubspot.py

@@ -0,0 +1,70 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliMixin, DelimitedString, Dict
+from unstructured_ingest.connector.hubspot import HubSpotObjectTypes, SimpleHubSpotConfig
+
+OBJECT_TYPES = {t.value for t in HubSpotObjectTypes}
+
+
+def validate_custom_property(ctx, param, value) -> t.Dict[str, t.List[str]]:
+    if not value:
+        return value
+    for k in value:
+        if k not in OBJECT_TYPES:
+            raise ValueError(f"Invalid object type: {k}, must be one of {OBJECT_TYPES}")
+        if not isinstance(value[k], list):
+            raise ValueError(f"Invalid type: {type(value[k])}, must be a Python list.")
+    return value
+
+
+@dataclass
+class HubSpotCliConfig(SimpleHubSpotConfig, CliMixin):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-token"],
+                required=True,
+                type=str,
+                help="Access token to perform operations on Hubspot. \
+                    Check \
+                    https://developers.hubspot.com/docs/api/private-apps/ \
+                    for more info",
+            ),
+            click.Option(
+                ["--object-types"],
+                default=None,
+                required=False,
+                type=DelimitedString(choices=OBJECT_TYPES),
+                is_flag=False,
+                help=f"Object to include in the process.\
+                    Must be a subset of {','.join(OBJECT_TYPES)}.\
+                    If the argument is omitted all objects listed will be processed.",
+            ),
+            click.Option(
+                ["--custom-properties"],
+                default=None,
+                required=False,
+                type=Dict(),
+                is_flag=False,
+                callback=validate_custom_property,
+                help="Custom property to process information from.\
+                    It should be a json-like string in the form\
+                        <object_type>:[<custom_property_id>, ..., <custom_property_id>]\
+                    Must be internal name of the variable. If the property is missing, \
+                        it will be omitted.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="hubspot",
+        cli_config=HubSpotCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/jira.py

@@ -0,0 +1,71 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import (
+    CliConfig,
+    DelimitedString,
+)
+from unstructured_ingest.connector.jira import SimpleJiraConfig
+
+
+@dataclass
+class JiraCliConfig(SimpleJiraConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--api-token"],
+                required=True,
+                type=str,
+                help="API Token to authenticate into Jira (into Atlassian). \
+                    Check \
+                    https://developer.atlassian.com/cloud/jira/platform/basic-auth-for-rest-apis/ \
+                    for more info.",
+            ),
+            click.Option(
+                ["--url"],
+                required=True,
+                type=str,
+                help="URL to Atlassian (Jira) Cloud, e.g. "
+                '"unstructured-jira-connector-test.atlassian.net"',
+            ),
+            click.Option(
+                ["--user-email"],
+                required=True,
+                type=str,
+                help="Email to authenticate into Atlassian (Jira) Cloud.",
+            ),
+            click.Option(
+                ["--projects"],
+                default=None,
+                type=DelimitedString(),
+                help="Comma-delimited Project ids or keys. Use Jira UI or the "
+                "API to find or obtain keys. Alternatively, use API to obtain ids.",
+            ),
+            click.Option(
+                ["--boards"],
+                default=None,
+                type=DelimitedString(),
+                help="Comma-delimited Board ids. Check board URL, or use the "
+                "API to find the board ids.",
+            ),
+            click.Option(
+                ["--issues"],
+                default=None,
+                type=DelimitedString(),
+                help="Comma-delimited Issue ids or keys. Use Jira UI or the API to "
+                "find or obtain keys. Alternatively, use API to obtain ids.",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name="jira",
+        cli_config=JiraCliConfig,
+    )
+    return cmd_cls

unstructured_ingest/cli/cmds/kafka.py

@@ -0,0 +1,102 @@
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.src import BaseSrcCmd
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.connector.kafka import KafkaWriteConfig, SimpleKafkaConfig
+
+CMD_NAME = "kafka"
+
+
+@dataclass
+class KafkaCliConfig(SimpleKafkaConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--bootstrap-server"], required=True, type=str, help="Broker server hostname"
+            ),
+            click.Option(
+                ["--port"],
+                required=True,
+                type=str,
+                help="The bootstrap port",
+            ),
+            click.Option(
+                ["--topic"],
+                required=True,
+                type=str,
+                help="The topic to write into.'",
+            ),
+            click.Option(
+                ["--kafka-api-key"],
+                required=False,
+                type=str,
+                help="The API KEY",
+            ),
+            click.Option(
+                ["--secret"],
+                required=False,
+                type=str,
+                help="The secret",
+            ),
+            click.Option(
+                ["--num-messages-to-consume"],
+                required=False,
+                type=int,
+                default=1,
+                help="The number of messages to consume before unblocking the consumer",
+            ),
+            click.Option(
+                ["--timeout"],
+                required=False,
+                type=float,
+                default=1.0,
+                help="Maximum time to block waiting for message(Seconds)",
+            ),
+            click.Option(
+                ["--confluent"],
+                required=False,
+                type=bool,
+                default=True,
+                help="Whether this Kafka instance is from Confluent",
+            ),
+        ]
+        return options
+
+
+@dataclass
+class KafkaCliWriteConfig(KafkaWriteConfig, CliConfig):
+    @staticmethod
+    def get_cli_options() -> t.List[click.Option]:
+        options = [
+            click.Option(
+                ["--batch-size"],
+                default=4,
+                type=int,
+                help="Number of records per batch",
+            ),
+        ]
+        return options
+
+
+def get_base_src_cmd() -> BaseSrcCmd:
+    cmd_cls = BaseSrcCmd(
+        cmd_name=CMD_NAME,
+        cli_config=KafkaCliConfig,
+    )
+    return cmd_cls
+
+
+def get_base_dest_cmd():
+    from unstructured_ingest.cli.base.dest import BaseDestCmd
+
+    cmd_cls = BaseDestCmd(
+        cmd_name=CMD_NAME,
+        cli_config=KafkaCliConfig,
+        additional_cli_options=[KafkaCliWriteConfig],
+        write_config=KafkaWriteConfig,
+    )
+    return cmd_cls