unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/cli/cmds/pinecone.py

@@ -1,62 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.pinecone import CONNECTOR_TYPE
-
-
-@dataclass
-class PineconeCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--api-key"],
-                required=True,
-                type=str,
-                help="API key for Pinecone.",
-            ),
-            click.Option(
-                ["--index-name"],
-                required=True,
-                type=str,
-                help="Name of the index to connect to. Example: my-index",
-            ),
-            click.Option(
-                ["--environment"],
-                required=True,
-                type=str,
-                help="Environment to connect to. Example: us-east-1",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class PineconeCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            ),
-            click.Option(
-                ["--num-processes"],
-                default=4,
-                type=int,
-                help="Number of processes to use for uploading",
-            ),
-        ]
-        return options
-
-
-pinecone_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=PineconeCliConnectionConfig,
-    uploader_config=PineconeCliUploaderConfig,
-)

unstructured_ingest/v2/cli/cmds/salesforce.py

@@ -1,79 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.salesforce import (
-    ACCEPTED_CATEGORIES,
-    CONNECTOR_TYPE,
-)
-
-
-@dataclass
-class SalesforceCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--username"],
-                required=True,
-                type=str,
-                help="Salesforce username usually looks like an email.",
-            ),
-            click.Option(
-                ["--consumer-key"],
-                required=True,
-                type=str,
-                help="For the Salesforce JWT auth. Found in Consumer Details.",
-            ),
-            click.Option(
-                ["--private-key"],
-                required=True,
-                type=str,
-                help="Path to the private key or its contents for the Salesforce JWT auth. "
-                "Key file is usually named server.key.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SalesforceCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        possible_categories = ACCEPTED_CATEGORIES
-        options = [
-            click.Option(
-                ["--categories"],
-                default=None,
-                required=True,
-                type=DelimitedString(choices=possible_categories),
-                help="Comma-delimited salesforce categories to download. "
-                "Currently only {}.".format(", ".join(possible_categories)),
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SalesforceCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-        return options
-
-
-salesforce_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SalesforceCliConnectionConfig,
-    indexer_config=SalesforceCliIndexerConfig,
-    downloader_config=SalesforceCliDownloadConfig,
-)

unstructured_ingest/v2/cli/cmds/sharepoint.py

@@ -1,112 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import SrcCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.sharepoint import CONNECTOR_TYPE
-
-
-@dataclass
-class SharepointCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--client-id"],
-                default=None,
-                type=str,
-                help="Sharepoint app client ID",
-            ),
-            click.Option(
-                ["--client-cred"],
-                default=None,
-                type=str,
-                help="Sharepoint app secret",
-            ),
-            click.Option(
-                ["--site"],
-                default=None,
-                type=str,
-                help="Sharepoint site url. Process either base url e.g \
-                    https://[tenant].sharepoint.com  or relative sites \
-                    https://[tenant].sharepoint.com/sites/<site_name>. \
-                    To process all sites within the tenant pass a site url as \
-                    https://[tenant]-admin.sharepoint.com.\
-                    This requires the app to be registered at a tenant level",
-            ),
-            click.Option(
-                ["--permissions-application-id"],
-                type=str,
-                help="Microsoft Graph API application id",
-            ),
-            click.Option(
-                ["--permissions-client-cred"],
-                type=str,
-                help="Microsoft Graph API application credentials",
-            ),
-            click.Option(
-                ["--permissions-tenant"],
-                type=str,
-                help="e.g https://contoso.onmicrosoft.com to get permissions data within tenant.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SharepointCliIndexerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--path"],
-                default=None,
-                type=str,
-                help="Path from which to start parsing files. If the connector is to \
-                process all sites within the tenant this filter will be applied to \
-                all sites document libraries.",
-            ),
-            click.Option(
-                ["--recursive"],
-                is_flag=True,
-                default=False,
-                help="Recursively download files in their respective folders "
-                "otherwise stop at the files in provided folder level.",
-            ),
-            click.Option(
-                ["--omit-files"],
-                is_flag=True,
-                default=False,
-                help="Don't process files.",
-            ),
-            click.Option(
-                ["--omit-pages"],
-                is_flag=True,
-                default=False,
-                help="Don't process site pages.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SharepointCliDownloadConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--download-dir"],
-                help="Where files are downloaded to, defaults to a location at"
-                "`$HOME/.cache/unstructured_ingest/<connector name>/<SHA256>`.",
-            ),
-        ]
-        return options
-
-
-sharepoint_drive_src_cmd = SrcCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SharepointCliConnectionConfig,
-    indexer_config=SharepointCliIndexerConfig,
-    downloader_config=SharepointCliDownloadConfig,
-)

unstructured_ingest/v2/cli/cmds/singlestore.py

@@ -1,96 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.singlestore import CONNECTOR_TYPE
-
-
-@dataclass
-class SingleStoreCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--host"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore host",
-            ),
-            click.Option(
-                ["--port"],
-                required=False,
-                type=int,
-                default=None,
-                help="SingleStore port",
-            ),
-            click.Option(
-                ["--user"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore user",
-            ),
-            click.Option(
-                ["--password"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore password",
-            ),
-            click.Option(
-                ["--database"],
-                required=False,
-                type=str,
-                default=None,
-                help="SingleStore database",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SingleStoreCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--drop-empty-cols"],
-                required=False,
-                type=bool,
-                is_flag=True,
-                default=False,
-                help="Drop any columns that have no data",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SingleStoreCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return [
-            click.Option(
-                ["--table-name"],
-                required=False,
-                type=str,
-                help="SingleStore table to write contents to",
-            ),
-            click.Option(
-                ["--batch-size"],
-                required=False,
-                type=click.IntRange(min=1),
-                help="Batch size when writing to SingleStore",
-            ),
-        ]
-
-
-singlestore_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SingleStoreCliConnectionConfig,
-    uploader_config=SingleStoreCliUploaderConfig,
-    upload_stager_config=SingleStoreCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/sql.py

@@ -1,84 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.processes.connectors.sql import CONNECTOR_TYPE
-
-SQL_DRIVERS = {"postgresql", "sqlite"}
-
-
-@dataclass
-class SQLCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--db-type"],
-                required=True,
-                type=click.Choice(SQL_DRIVERS),
-                help="Type of the database backend",
-            ),
-            click.Option(
-                ["--username"],
-                default=None,
-                type=str,
-                help="DB username",
-            ),
-            click.Option(
-                ["--password"],
-                default=None,
-                type=str,
-                help="DB password",
-            ),
-            click.Option(
-                ["--host"],
-                default=None,
-                type=str,
-                help="DB host",
-            ),
-            click.Option(
-                ["--port"],
-                default=None,
-                type=int,
-                help="DB host connection port",
-            ),
-            click.Option(
-                ["--database"],
-                default=None,
-                type=str,
-                help="Database name. For sqlite databases, this is the path to the .db file.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class SQLCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-@dataclass
-class SQLCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-sql_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=SQLCliConnectionConfig,
-    uploader_config=SQLCliUploaderConfig,
-    upload_stager_config=SQLCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/cmds/weaviate.py

@@ -1,100 +0,0 @@
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.v2.cli.base import DestCmd
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-from unstructured_ingest.v2.cli.utils import DelimitedString
-from unstructured_ingest.v2.processes.connectors.weaviate import CONNECTOR_TYPE
-
-
-@dataclass
-class WeaviateCliConnectionConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--host-url"],
-                required=True,
-                help="Weaviate instance url",
-            ),
-            click.Option(
-                ["--class-name"],
-                default=None,
-                type=str,
-                help="Name of the class to push the records into, e.g: Pdf-elements",
-            ),
-            click.Option(
-                ["--access-token"], default=None, type=str, help="Used to create the bearer token."
-            ),
-            click.Option(
-                ["--refresh-token"],
-                default=None,
-                type=str,
-                help="Will tie this value to the bearer token. If not provided, "
-                "the authentication will expire once the lifetime of the access token is up.",
-            ),
-            click.Option(
-                ["--api-key"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--client-secret"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--scope"],
-                default=None,
-                type=DelimitedString(),
-            ),
-            click.Option(
-                ["--username"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--password"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--anonymous"],
-                is_flag=True,
-                default=False,
-                type=bool,
-                help="if set, all auth values will be ignored",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class WeaviateCliUploaderConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-@dataclass
-class WeaviateCliUploadStagerConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        return []
-
-
-weaviate_dest_cmd = DestCmd(
-    cmd_name=CONNECTOR_TYPE,
-    connection_config=WeaviateCliConnectionConfig,
-    uploader_config=WeaviateCliUploaderConfig,
-    upload_stager_config=WeaviateCliUploadStagerConfig,
-)

unstructured_ingest/v2/cli/configs/__init__.py

@@ -1,13 +0,0 @@
-from .chunk import ChunkerCliConfig
-from .embed import EmbedderCliConfig
-from .filter import FilterCliConfig
-from .partition import PartitionerCliConfig
-from .processor import ProcessorCliConfig
-
-__all__ = [
-    "ChunkerCliConfig",
-    "ProcessorCliConfig",
-    "PartitionerCliConfig",
-    "EmbedderCliConfig",
-    "FilterCliConfig",
-]

unstructured_ingest/v2/cli/configs/chunk.py

@@ -1,89 +0,0 @@
-from dataclasses import dataclass
-
-import click
-from unstructured.chunking import CHUNK_MAX_CHARS_DEFAULT, CHUNK_MULTI_PAGE_DEFAULT
-
-from unstructured_ingest.v2.cli.interfaces import CliConfig
-
-
-@dataclass
-class ChunkerCliConfig(CliConfig):
-    @staticmethod
-    def get_cli_options() -> list[click.Option]:
-        options = [
-            click.Option(
-                ["--chunking-strategy"],
-                type=str,
-                default=None,
-                help="The rule-set to use to form chunks. Omit to disable chunking.",
-            ),
-            click.Option(
-                ["--chunk-combine-text-under-n-chars"],
-                type=int,
-                help=(
-                    "Combine consecutive chunks when the first does not exceed this length and"
-                    " the second will fit without exceeding the hard-maximum length. Only"
-                    " operative for 'by_title' chunking-strategy."
-                ),
-            ),
-            click.Option(
-                ["--chunk-include-orig-elements/--chunk-no-include-orig-elements"],
-                is_flag=True,
-                default=True,
-                help=(
-                    "When chunking, add the original elements consolidated to form each chunk to"
-                    " `.metadata.orig_elements` on that chunk."
-                ),
-            ),
-            click.Option(
-                ["--chunk-max-characters"],
-                type=int,
-                default=CHUNK_MAX_CHARS_DEFAULT,
-                show_default=True,
-                help=(
-                    "Hard maximum chunk length. No chunk will exceed this length. An oversized"
-                    " element will be divided by text-splitting to fit this window."
-                ),
-            ),
-            click.Option(
-                ["--chunk-multipage-sections/--chunk-no-multipage-sections"],
-                is_flag=True,
-                default=CHUNK_MULTI_PAGE_DEFAULT,
-                help=(
-                    "Ignore page boundaries when chunking such that elements from two different"
-                    " pages can appear in the same chunk. Only operative for 'by_title'"
-                    " chunking-strategy."
-                ),
-            ),
-            click.Option(
-                ["--chunk-new-after-n-chars"],
-                type=int,
-                help=(
-                    "Soft-maximum chunk length. Another element will not be added to a chunk of"
-                    " this length even when it would fit without exceeding the hard-maximum"
-                    " length."
-                ),
-            ),
-            click.Option(
-                ["--chunk-overlap"],
-                type=int,
-                default=0,
-                show_default=True,
-                help=(
-                    "Prefix chunk text with last overlap=N characters of prior chunk. Only"
-                    " applies to oversized chunks divided by text-splitting. To apply overlap to"
-                    " non-oversized chunks use the --overlap-all option."
-                ),
-            ),
-            click.Option(
-                ["--chunk-overlap-all"],
-                is_flag=True,
-                default=False,
-                help=(
-                    "Apply overlap to chunks formed from whole elements as well as those formed"
-                    " by text-splitting oversized elements. Overlap length is take from --overlap"
-                    " option value."
-                ),
-            ),
-        ]
-        return options