unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/cli/cmds/fsspec/fsspec.py

@@ -1,77 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-
-
-@dataclass
-class FsspecCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-
-
-@dataclass
-class FsspecCliFileConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return [
-            click.Option(
-                ["--remote-url"],
-                required=True,
-                help="Remote fsspec URL formatted as `protocol://dir/path`",
-            )
-        ]
-
-
-@dataclass
-class FsspecCliUploaderConfig(FsspecCliFileConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = super(FsspecCliUploaderConfig, FsspecCliUploaderConfig).get_cli_options()
-        options.extend(
-            [
-                click.Option(
-                    ["--overwrite"],
-                    is_flag=True,
-                    default=False,
-                    show_default=True,
-                    help="If set, will overwrite content if content already exists",
-                )
-            ]
-        )
-        return options
-
-
-@dataclass
-class FsspecCliIndexerConfig(FsspecCliFileConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = super(FsspecCliIndexerConfig, FsspecCliIndexerConfig).get_cli_options()
-        options.extend(
-            [
-                click.Option(
-                    ["--recursive"],
-                    is_flag=True,
-                    default=False,
-                    help="Recursively download files in their respective folders "
-                    "otherwise stop at the files in provided folder level.",
-                ),
-                click.Option(
-                    ["--file-glob"],
-                    default=None,
-                    type=DelimitedString(),
-                    help="A comma-separated list of file globs to limit which types of "
-                    "local files are accepted, e.g. '*.html,*.txt'",
-                ),
-            ]
-        )
-        return options

unstructured_ingest/v2/cli/cmds/fsspec/gcs.py

@@ -1,81 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.fsspec.fsspec import (
-    FsspecCliDownloadConfig,
-    FsspecCliIndexerConfig,
-    FsspecCliUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import FileOrJson
-from unstructured_ingest.v2.processes.connectors.fsspec.gcs import (
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class GcsCliDownloadConfig(FsspecCliDownloadConfig):
-    pass
-
-
-@dataclass
-class GcsCliIndexerConfig(FsspecCliIndexerConfig):
-    pass
-
-
-@dataclass
-class GcsCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        help_string = """
-          Options:
-          - ``None``, GCSFS will attempt to guess your credentials in the
-        following order: gcloud CLI default, gcsfs cached token, google compute
-        metadata service, anonymous.
-          - ``'google_default'``, your default gcloud credentials will be used,
-            which are typically established by doing ``gcloud login`` in a terminal.
-          - ``'cache'``, credentials from previously successful gcsfs
-            authentication will be used (use this after "browser" auth succeeded)
-          - ``'anon'``, no authentication is performed, and you can only
-            access data which is accessible to allUsers (in this case, the project and
-            access level parameters are meaningless)
-          - ``'browser'``, you get an access code with which you can
-            authenticate via a specially provided URL
-          - if ``'cloud'``, we assume we are running within google compute
-            or google container engine, and query the internal metadata directly for
-            a token.
-          - you may supply a token generated by the
-            [gcloud](https://cloud.google.com/sdk/docs/)
-            utility; this is either a python dictionary or the name of a file
-            containing the JSON returned by logging in with the gcloud CLI tool.
-          """
-        options = [
-            click.Option(
-                ["--service-account-key"],
-                default=None,
-                type=FileOrJson(allow_raw_str=True),
-                help=help_string,
-            ),
-        ]
-        return options
-
-
-@dataclass
-class GcsUploaderConfig(FsspecCliUploaderConfig):
-    pass
-
-
-gcs_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=GcsCliIndexerConfig,
-    connection_config=GcsCliConnectionConfig,
-    downloader_config=GcsCliDownloadConfig,
-)
-
-gcs_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=GcsCliConnectionConfig,
-    uploader_config=GcsUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/fsspec/s3.py

@@ -1,84 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.fsspec.fsspec import (
-    FsspecCliDownloadConfig,
-    FsspecCliIndexerConfig,
-    FsspecCliUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class S3CliDownloadConfig(FsspecCliDownloadConfig):
-    pass
-
-
-@dataclass
-class S3CliIndexerConfig(FsspecCliIndexerConfig):
-    pass
-
-
-@dataclass
-class S3CliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--anonymous"],
-                is_flag=True,
-                default=False,
-                help="Connect to s3 without local AWS credentials.",
-            ),
-            click.Option(
-                ["--endpoint-url"],
-                type=str,
-                default=None,
-                help="Use this endpoint_url, if specified. Needed for "
-                "connecting to non-AWS S3 buckets.",
-            ),
-            click.Option(
-                ["--key"],
-                type=str,
-                default=None,
-                help="If not anonymous, use this access key ID, if specified. Takes precedence "
-                "over `aws_access_key_id` in client_kwargs.",
-            ),
-            click.Option(
-                ["--secret"],
-                type=str,
-                default=None,
-                help="If not anonymous, use this secret access key, if specified.",
-            ),
-            click.Option(
-                ["--token"],
-                type=str,
-                default=None,
-                help="If not anonymous, use this security token, if specified.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class S3UploaderConfig(FsspecCliUploaderConfig):
-    pass
-
-
-s3_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=S3CliIndexerConfig,
-    connection_config=S3CliConnectionConfig,
-    downloader_config=S3CliDownloadConfig,
-)
-
-s3_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=S3CliConnectionConfig,
-    uploader_config=S3UploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/fsspec/sftp.py

@@ -1,80 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.cmds.fsspec.fsspec import (
-    FsspecCliDownloadConfig,
-    FsspecCliIndexerConfig,
-    FsspecCliUploaderConfig,
-)
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.fsspec.sftp import (
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class SftpCliDownloadConfig(FsspecCliDownloadConfig):
-    pass
-
-
-@dataclass
-class SftpCliIndexerConfig(FsspecCliIndexerConfig):
-    pass
-
-
-@dataclass
-class SftpCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--username"],
-                required=True,
-                type=str,
-                help="Username for sftp connection",
-            ),
-            click.Option(
-                ["--password"],
-                required=True,
-                type=str,
-                help="Password for sftp connection",
-            ),
-            click.Option(
-                ["--look-for-keys"],
-                required=False,
-                default=False,
-                is_flag=True,
-                type=bool,
-                help="Whether to search for private key files in ~/.ssh/",
-            ),
-            click.Option(
-                ["--allow-agent"],
-                required=False,
-                default=False,
-                is_flag=True,
-                type=bool,
-                help="Whether to connect to the SSH agent.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SftpUploaderConfig(FsspecCliUploaderConfig):
-    pass
-
-
-sftp_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=SftpCliIndexerConfig,
-    connection_config=SftpCliConnectionConfig,
-    downloader_config=SftpCliDownloadConfig,
-)
-
-sftp_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SftpCliConnectionConfig,
-    uploader_config=SftpUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/google_drive.py

@@ -1,74 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString, FileOrJson
-from unstructured_ingest.v2.processes.connectors.google_drive import CONNECTOR_TYPE
-
-
-@dataclass
-class GoogleDriveCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--drive-id"],
-                required=True,
-                type=str,
-                help="Google Drive File or Folder ID.",
-            ),
-            click.Option(
-                ["--service-account-key"],
-                required=True,
-                type=FileOrJson(),
-                help="Either the file path of the credentials file to use or a json string of "
-                "those values to use for authentication",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class GoogleDriveCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--extensions"],
-                default=None,
-                type=DelimitedString(),
-                help="Filters the files to be processed based on extension e.g. jpg, docx, etc.",
-            ),
-            click.Option(
-                ["--recursive"],
-                is_flag=True,
-                default=False,
-                help="Recursively download files in their respective folders "
-                "otherwise stop at the files in provided folder level.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class GoogleDriveCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-        return options
-
-
-google_drive_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=GoogleDriveCliConnectionConfig,
-    indexer_config=GoogleDriveCliIndexerConfig,
-    downloader_config=GoogleDriveCliDownloadConfig,
-)

unstructured_ingest/v2/cli/cmds/local.py

@@ -1,60 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd, SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.local import CONNECTOR_TYPE
-
-
-@dataclass
-class LocalCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--input-path"],
-                required=True,
-                type=click.Path(file_okay=True, dir_okay=True, exists=True),
-                help="Path to the location in the local file system that will be processed.",
-            ),
-            click.Option(
-                ["--file-glob"],
-                default=None,
-                type=DelimitedString(),
-                help="A comma-separated list of file globs to limit which types of "
-                "local files are accepted, e.g. '*.html,*.txt'",
-            ),
-            click.Option(
-                ["--recursive"],
-                is_flag=True,
-                default=False,
-                help="Recursively download files in their respective folders "
-                "otherwise stop at the files in provided folder level.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class LocalCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--output-dir"],
-                required=True,
-                type=str,
-                help="Local path to write partitioned output to",
-            )
-        ]
-        return options
-
-
-local_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    indexer_config=LocalCliIndexerConfig,
-)
-
-local_dest_cmd = DestCmd(cmd_name=CONNECTOR_TYPE, uploader_config=LocalCliUploaderConfig)

unstructured_ingest/v2/cli/cmds/milvus.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.milvus import CONNECTOR_TYPE
-
-
-@dataclass
-class MilvusCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--uri"],
-                required=False,
-                type=str,
-                default=None,
-                help="Milvus uri, eg 'http://localhost:19530",
-            ),
-            click.Option(
-                ["--user"],
-                required=False,
-                type=str,
-                default=None,
-                help="Milvus user",
-            ),
-            click.Option(
-                ["--password"],
-                required=False,
-                type=str,
-                default=None,
-                help="Milvus password",
-            ),
-            click.Option(
-                ["--db-name"],
-                required=False,
-                type=str,
-                default=None,
-                help="Milvus database name",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class MilvusCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--collection-name"],
-                required=True,
-                type=str,
-                help="Milvus collections to write to",
-            ),
-            click.Option(
-                ["--num-of-processes"],
-                type=click.IntRange(min=1),
-                default=4,
-                help="number of processes to use when writing to support parallel writes",
-            ),
-        ]
-        return options
-
-
-milvus_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=MilvusCliConnectionConfig,
-    uploader_config=MilvusCliUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/mongodb.py

@@ -1,62 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.mongodb import CONNECTOR_TYPE
-
-
-@dataclass
-class MongoDBCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--uri"],
-                help="URI to user when connecting",
-            ),
-            click.Option(
-                ["--host"],
-                help="hostname or IP address or Unix domain socket path of a single mongod or "
-                "mongos instance to connect to, or a list of hostnames",
-            ),
-            click.Option(["--port"], type=int, default=27017),
-            click.Option(
-                ["--database"], type=str, required=True, help="database name to connect to"
-            ),
-            click.Option(
-                ["--collection"], required=True, type=str, help="collection name to connect to"
-            ),
-        ]
-        return options
-
-
-@dataclass
-class MongoDBCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-@dataclass
-class MongoDBCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-mongodb_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=MongoDBCliConnectionConfig,
-    uploader_config=MongoDBCliUploaderConfig,
-    upload_stager_config=MongoDBCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/onedrive.py

@@ -1,91 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.onedrive import CONNECTOR_TYPE
-
-
-@dataclass
-class OnedriveCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                required=True,
-                type=str,
-                help="Microsoft app client ID",
-            ),
-            click.Option(
-                ["--client-cred"],
-                required=True,
-                type=str,
-                help="Microsoft App client secret",
-            ),
-            click.Option(
-                ["--user-pname"],
-                required=True,
-                type=str,
-                help="User principal name, usually is your Azure AD email.",
-            ),
-            click.Option(
-                ["--tenant"],
-                default="common",
-                type=str,
-                help="ID or domain name associated with your Azure AD instance",
-            ),
-            click.Option(
-                ["--authority-url"],
-                default="https://login.microsoftonline.com",
-                type=str,
-                help="Authentication token provider for Microsoft apps, default is "
-                "https://login.microsoftonline.com",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class OnedriveCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--path"],
-                default=None,
-                type=str,
-                help="Folder to start parsing files from.",
-            ),
-            click.Option(
-                ["--recursive"],
-                is_flag=True,
-                default=False,
-                help="Recursively download files in their respective folders "
-                "otherwise stop at the files in provided folder level.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class OnedriveCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-        return options
-
-
-onedrive_drive_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=OnedriveCliConnectionConfig,
-    indexer_config=OnedriveCliIndexerConfig,
-    downloader_config=OnedriveCliDownloadConfig,
-)