unstructured-ingest 0.6.2__py3-none-any.whl → 0.7.0__py3-none-any.whl

unstructured_ingest/cli/cmds/airtable.py

@@ -1,69 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.airtable import SimpleAirtableConfig
-
-
-@dataclass
-class AirtableCliConfig(SimpleAirtableConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--personal-access-token"],
-                default=None,
-                help="Personal access token to authenticate into Airtable. Check: "
-                "https://support.airtable.com/docs/creating-and-using-api-keys-and-access-tokens "
-                "for more info",
-            ),
-            click.Option(
-                ["--list-of-paths"],
-                default=None,
-                help="""
-        A list of paths that specify the locations to ingest data from within Airtable.
-
-        If this argument is not set, the connector ingests all tables within each and every base.
-        --list-of-paths: path1 path2 path3 ….
-        path: base_id/table_id(optional)/view_id(optional)/
-
-        To obtain (base, table, view) ids in bulk, check:
-        https://airtable.com/developers/web/api/list-bases (base ids)
-        https://airtable.com/developers/web/api/get-base-schema (table and view ids)
-        https://pyairtable.readthedocs.io/en/latest/metadata.html (base, table and view ids)
-
-        To obtain specific ids from Airtable UI, go to your workspace, and copy any
-        relevant id from the URL structure:
-        https://airtable.com/appAbcDeF1ghijKlm/tblABcdEfG1HIJkLm/viwABCDEfg6hijKLM
-        appAbcDeF1ghijKlm -> base_id
-        tblABcdEfG1HIJkLm -> table_id
-        viwABCDEfg6hijKLM -> view_id
-
-        You can also check: https://support.airtable.com/docs/finding-airtable-ids
-
-        Here is an example for one --list-of-paths:
-            base1/		→ gets the entirety of all tables inside base1
-            base1/table1		→ gets all rows and columns within table1 in base1
-            base1/table1/view1	→ gets the rows and columns that are
-                                  visible in view1 for the table1 in base1
-
-        Examples to invalid airtable_paths:
-            table1          → has to mention base to be valid
-            base1/view1     → has to mention table to be valid
-                """,
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="airtable",
-        cli_config=AirtableCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/astradb.py

@@ -1,99 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig, Dict
-from unstructured_ingest.connector.astradb import AstraDBWriteConfig, SimpleAstraDBConfig
-
-
-@dataclass
-class AstraDBCliConfig(SimpleAstraDBConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--token"],
-                required=True,
-                type=str,
-                help="Astra DB Token with access to the database.",
-                envvar="ASTRA_DB_APPLICATION_TOKEN",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--api-endpoint"],
-                required=True,
-                type=str,
-                help="The API endpoint for the Astra DB.",
-                envvar="ASTRA_DB_API_ENDPOINT",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--collection-name"],
-                required=False,
-                type=str,
-                help="The name of the Astra DB collection. "
-                "Note that the collection name must only include letters, "
-                "numbers, and underscores.",
-            ),
-            click.Option(
-                ["--keyspace"],
-                required=False,
-                default=None,
-                type=str,
-                help="The Astra DB connection keyspace.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class AstraDBCliWriteConfig(AstraDBWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--embedding-dimension"],
-                required=True,
-                default=384,
-                type=int,
-                help="The dimensionality of the embeddings",
-            ),
-            click.Option(
-                ["--requested-indexing-policy"],
-                required=False,
-                default=None,
-                type=Dict(),
-                help="The indexing policy to use for the collection."
-                'example: \'{"deny": ["metadata"]}\' ',
-            ),
-            click.Option(
-                ["--batch-size"],
-                default=20,
-                type=int,
-                help="Number of records per batch",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd():
-    from unstructured_ingest.cli.base.src import BaseSrcCmd
-
-    cmd_cls = BaseSrcCmd(
-        cmd_name="astradb",
-        cli_config=AstraDBCliConfig,
-    )
-    return cmd_cls
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="astradb",
-        cli_config=AstraDBCliConfig,
-        additional_cli_options=[AstraDBCliWriteConfig],
-        write_config=AstraDBWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/azure_ai_search.py

@@ -1,65 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.azure_ai_search import (
-    AzureAISearchWriteConfig,
-    SimpleAzureAISearchStorageConfig,
-)
-
-
-@dataclass
-class AzureAISearchCliConfig(SimpleAzureAISearchStorageConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--key"],
-                required=True,
-                type=str,
-                help="Key credential used for authenticating to an Azure service.",
-                envvar="AZURE_SEARCH_API_KEY",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--endpoint"],
-                required=True,
-                type=str,
-                help="The URL endpoint of an Azure search service. "
-                "In the form of https://{{service_name}}.search.windows.net",
-                envvar="AZURE_SEARCH_ENDPOINT",
-                show_envvar=True,
-            ),
-        ]
-        return options
-
-
-@dataclass
-class AzureAISearchCliWriteConfig(AzureAISearchWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--index"],
-                required=True,
-                type=str,
-                help="The name of the index to connect to",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="azure-ai-search",
-        cli_config=AzureAISearchCliConfig,
-        additional_cli_options=[AzureAISearchCliWriteConfig],
-        write_config=AzureAISearchCliWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/biomed.py

@@ -1,52 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.biomed import SimpleBiomedConfig
-
-
-@dataclass
-class BiomedCliConfig(SimpleBiomedConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--api-id"],
-                default=None,
-                help="ID parameter for OA Web Service API.",
-            ),
-            click.Option(
-                ["--api-from"],
-                default=None,
-                help="From parameter for OA Web Service API.",
-            ),
-            click.Option(
-                ["--api-until"],
-                default=None,
-                help="Until parameter for OA Web Service API.",
-            ),
-            click.Option(
-                ["--path"],
-                default=None,
-                help="PMC Open Access FTP Directory Path.",
-            ),
-            click.Option(
-                ["--max-request-time"],
-                default=45,
-                help="(In seconds) Max request time to OA Web Service API.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="biomed",
-        cli_config=BiomedCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/chroma.py

@@ -1,104 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig, Dict
-from unstructured_ingest.connector.chroma import ChromaWriteConfig, SimpleChromaConfig
-
-
-@dataclass
-class ChromaCliConfig(SimpleChromaConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--path"],
-                required=False,
-                type=str,
-                help="Location where Chroma is persisted," "if not connecting via http.",
-            ),
-            click.Option(
-                ["--settings"],
-                required=False,
-                type=Dict(),
-                help="A dictionary of settings to communicate with the chroma server."
-                'example: \'{"persist_directory":"./chroma-persist"}\' ',
-            ),
-            click.Option(
-                ["--tenant"],
-                required=False,
-                default="default_tenant",
-                type=str,
-                help="The tenant to use for this client. Chroma defaults to 'default_tenant'.",
-            ),
-            click.Option(
-                ["--database"],
-                required=False,
-                default="default_database",
-                type=str,
-                help="The database to use for this client."
-                "Chroma defaults to 'default_database'.",
-            ),
-            click.Option(
-                ["--host"],
-                required=False,
-                type=str,
-                help="The hostname of the Chroma server.",
-            ),
-            click.Option(
-                ["--port"],
-                required=False,
-                type=int,
-                help="The port of the Chroma server.",
-            ),
-            click.Option(
-                ["--ssl"],
-                required=False,
-                default=False,
-                is_flag=True,
-                type=bool,
-                help="Whether to use SSL to connect to the Chroma server.",
-            ),
-            click.Option(
-                ["--headers"],
-                required=False,
-                type=Dict(),
-                help="A dictionary of headers to send to the Chroma server."
-                'example: \'{"Authorization":"Basic()"}\' ',
-            ),
-            click.Option(
-                ["--collection-name"],
-                required=True,
-                type=str,
-                help="The name of the Chroma collection to write into.",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ChromaCliWriteConfig(ChromaWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="chroma",
-        cli_config=ChromaCliConfig,
-        additional_cli_options=[ChromaCliWriteConfig],
-        write_config=ChromaWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/clarifai.py

@@ -1,71 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig
-from unstructured_ingest.connector.clarifai import (
-    ClarifaiWriteConfig,
-    SimpleClarifaiConfig,
-)
-
-CMD_NAME = "clarifai"
-
-
-@dataclass
-class ClarifaiCliConfig(SimpleClarifaiConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--api-key"],
-                required=True,
-                type=str,
-                help="The CLARIFAI_PAT of the user to access clarifai platform apps and models",
-                envvar="CLARIFAI_PAT",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--app-id"],
-                required=True,
-                type=str,
-                help="Clarifai app name/id",
-            ),
-            click.Option(
-                ["--user-id"],
-                required=True,
-                type=str,
-                help="Clarifai User name/ID",
-            ),
-            click.Option(
-                ["--dataset-id"], type=str, default=None, help="Clarifai App Dataset ID (optional)"
-            ),
-        ]
-        return options
-
-
-@dataclass
-class ClarifaiCliWriteConfig(ClarifaiWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                type=int,
-                default=50,
-                help="No of inputs upload per batch",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=ClarifaiCliConfig,
-        additional_cli_options=[ClarifaiCliWriteConfig],
-        write_config=ClarifaiWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/confluence.py

@@ -1,69 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-    DelimitedString,
-)
-from unstructured_ingest.connector.confluence import SimpleConfluenceConfig
-
-
-@dataclass
-class ConfluenceCliConfig(SimpleConfluenceConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--api-token"],
-                required=True,
-                help="API Token to authenticate into Confluence Cloud. "
-                "Check "
-                "https://developer.atlassian.com/cloud/confluence/basic-auth-for-rest-apis/ "
-                "for more info.",
-            ),
-            click.Option(
-                ["--url"],
-                required=True,
-                help='URL to Confluence Cloud, e.g. "unstructured-ingest-test.atlassian.net"',
-            ),
-            click.Option(
-                ["--user-email"],
-                required=True,
-                help="Email to authenticate into Confluence Cloud",
-            ),
-            click.Option(
-                ["--spaces"],
-                default=None,
-                type=DelimitedString(),
-                help="A list of confluence space ids to be fetched. From each fetched space, "
-                "--num-of-docs-from-each-space number of docs will be ingested. "
-                "--spaces and --num-of-spaces cannot be used at the same time",
-            ),
-            click.Option(
-                ["--max-num-of-docs-from-each-space"],
-                default=100,
-                help="Number of documents to be aimed to be ingested from each fetched "
-                "confluence space. If any space has fewer documents, all the documents from "
-                "that space will be ingested. Documents are not necessarily "
-                "ingested in order of creation date.",
-            ),
-            click.Option(
-                ["--max-num-of-spaces"],
-                default=500,
-                help="Number of confluence space ids to be fetched. From each fetched space, "
-                "--num-of-docs-from-each-space number of docs will be ingested. "
-                "--spaces and --num-of-spaces cannot be used at the same time",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="confluence",
-        cli_config=ConfluenceCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/databricks_volumes.py

@@ -1,163 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig
-from unstructured_ingest.connector.databricks_volumes import (
-    DatabricksVolumesWriteConfig,
-    SimpleDatabricksVolumesConfig,
-)
-
-CMD_NAME = "databricks-volumes"
-
-
-@dataclass
-class DatabricksVolumesCliConfig(SimpleDatabricksVolumesConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--host"],
-                type=str,
-                default=None,
-                help="The Databricks host URL for either the "
-                "Databricks workspace endpoint or the "
-                "Databricks accounts endpoint.",
-            ),
-            click.Option(
-                ["--account-id"],
-                type=str,
-                default=None,
-                help="The Databricks account ID for the Databricks "
-                "accounts endpoint. Only has effect when Host is "
-                "either https://accounts.cloud.databricks.com/ (AWS), "
-                "https://accounts.azuredatabricks.net/ (Azure), "
-                "or https://accounts.gcp.databricks.com/ (GCP).",
-            ),
-            click.Option(
-                ["--username"],
-                type=str,
-                default=None,
-                help="The Databricks username part of basic authentication. "
-                "Only possible when Host is *.cloud.databricks.com (AWS).",
-            ),
-            click.Option(
-                ["--password"],
-                type=str,
-                default=None,
-                help="The Databricks password part of basic authentication. "
-                "Only possible when Host is *.cloud.databricks.com (AWS).",
-            ),
-            click.Option(["--client-id"], type=str, default=None),
-            click.Option(["--client-secret"], type=str, default=None),
-            click.Option(
-                ["--token"],
-                type=str,
-                default=None,
-                help="The Databricks personal access token (PAT) (AWS, Azure, and GCP) or "
-                "Azure Active Directory (Azure AD) token (Azure).",
-            ),
-            click.Option(
-                ["--azure-workspace-resource-id"],
-                type=str,
-                default=None,
-                help="The Azure Resource Manager ID for the Azure Databricks workspace, "
-                "which is exchanged for a Databricks host URL.",
-            ),
-            click.Option(
-                ["--azure-client-secret"],
-                type=str,
-                default=None,
-                help="The Azure AD service principal’s client secret.",
-            ),
-            click.Option(
-                ["--azure-client-id"],
-                type=str,
-                default=None,
-                help="The Azure AD service principal’s application ID.",
-            ),
-            click.Option(
-                ["--azure-tenant-id"],
-                type=str,
-                default=None,
-                help="The Azure AD service principal’s tenant ID.",
-            ),
-            click.Option(
-                ["--azure-environment"],
-                type=str,
-                default=None,
-                help="The Azure environment type (such as Public, UsGov, China, and Germany) for a "
-                "specific set of API endpoints. Defaults to PUBLIC.",
-            ),
-            click.Option(
-                ["--auth-type"],
-                type=str,
-                default=None,
-                help="When multiple auth attributes are available in the "
-                "environment, use the auth type specified by this "
-                "argument. This argument also holds the currently "
-                "selected auth.",
-            ),
-            click.Option(["--cluster-id"], type=str, default=None),
-            click.Option(["--google-credentials"], type=str, default=None),
-            click.Option(["--google-service-account"], type=str, default=None),
-        ]
-        return options
-
-
-@dataclass
-class DatabricksVolumesCliWriteConfig(DatabricksVolumesWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--volume"], type=str, required=True, help="Name of volume in the Unity Catalog"
-            ),
-            click.Option(
-                ["--catalog"],
-                type=str,
-                required=True,
-                help="Name of the catalog in the Databricks Unity Catalog service",
-            ),
-            click.Option(
-                ["--volume-path"],
-                type=str,
-                required=False,
-                default=None,
-                help="Optional path within the volume to write to",
-            ),
-            click.Option(
-                ["--overwrite"],
-                type=bool,
-                is_flag=True,
-                help="If true, an existing file will be overwritten.",
-            ),
-            click.Option(
-                ["--encoding"],
-                type=str,
-                required=True,
-                default="utf-8",
-                help="Encoding applied to the data when written to the volume",
-            ),
-            click.Option(
-                ["--schema"],
-                type=str,
-                required=True,
-                default="default",
-                help="Schema associated with the volume to write to in the Unity Catalog service",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=DatabricksVolumesCliConfig,
-        additional_cli_options=[DatabricksVolumesCliWriteConfig],
-        write_config=DatabricksVolumesWriteConfig,
-    )
-    return cmd_cls