unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.1__py3-none-any.whl

unstructured_ingest/cli/cmds/vectara.py

@@ -1,66 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig
-from unstructured_ingest.connector.vectara import SimpleVectaraConfig, WriteConfig
-
-
-@dataclass
-class VectaraCliWriteConfig(SimpleVectaraConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--customer-id"],
-                required=True,
-                type=str,
-                help="The Vectara customer-id.",
-                envvar="VECTARA_CUSTOMER_ID",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--oauth-client-id"],
-                required=True,
-                type=str,
-                help="Vectara OAuth2 client ID.",
-                envvar="VECTARA_OAUTH_CLIENT_ID",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--oauth-secret"],
-                required=True,
-                type=str,
-                help="Vectara OAuth2 secret.",
-                envvar="VECTARA_OAUTH_SECRET",
-                show_envvar=True,
-            ),
-            click.Option(
-                ["--corpus-name"],
-                required=False,
-                type=str,
-                default=None,
-                help="The Vectara corpus-name.",
-            ),
-            click.Option(
-                ["--token-url"],
-                required=False,
-                default="https://vectara-prod-{}.auth.us-west-2.amazoncognito.com/oauth2/token",
-                type=str,
-                help="The Vectara endpoint for token refresh. Needs curly brackets for customer_id",
-            ),
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name="vectara",
-        cli_config=VectaraCliWriteConfig,
-        additional_cli_options=[],
-        write_config=WriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/weaviate.py

@@ -1,98 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.interfaces import CliConfig, DelimitedString
-from unstructured_ingest.connector.weaviate import SimpleWeaviateConfig, WeaviateWriteConfig
-
-CMD_NAME = "weaviate"
-
-
-@dataclass
-class WeaviateCliConfig(SimpleWeaviateConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--host-url"],
-                required=True,
-                help="Weaviate instance url",
-            ),
-            click.Option(
-                ["--class-name"],
-                default=None,
-                type=str,
-                help="Name of the class to push the records into, e.g: Pdf-elements",
-            ),
-            click.Option(
-                ["--access-token"], default=None, type=str, help="Used to create the bearer token."
-            ),
-            click.Option(
-                ["--refresh-token"],
-                default=None,
-                type=str,
-                help="Will tie this value to the bearer token. If not provided, "
-                "the authentication will expire once the lifetime of the access token is up.",
-            ),
-            click.Option(
-                ["--api-key"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--client-secret"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--scope"],
-                default=None,
-                type=DelimitedString(),
-            ),
-            click.Option(
-                ["--username"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--password"],
-                default=None,
-                type=str,
-            ),
-            click.Option(
-                ["--anonymous"],
-                is_flag=True,
-                default=False,
-                type=bool,
-                help="if set, all auth values will be ignored",
-            ),
-        ]
-        return options
-
-
-@dataclass
-class WeaviateCliWriteConfig(WeaviateWriteConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--batch-size"],
-                default=100,
-                type=int,
-                help="Number of records per batch",
-            )
-        ]
-        return options
-
-
-def get_base_dest_cmd():
-    from unstructured_ingest.cli.base.dest import BaseDestCmd
-
-    cmd_cls = BaseDestCmd(
-        cmd_name=CMD_NAME,
-        cli_config=WeaviateCliConfig,
-        additional_cli_options=[WeaviateCliWriteConfig],
-        write_config=WeaviateWriteConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/cmds/wikipedia.py

@@ -1,40 +0,0 @@
-import typing as t
-from dataclasses import dataclass
-
-import click
-
-from unstructured_ingest.cli.base.src import BaseSrcCmd
-from unstructured_ingest.cli.interfaces import (
-    CliConfig,
-)
-from unstructured_ingest.connector.wikipedia import SimpleWikipediaConfig
-
-
-@dataclass
-class WikipediaCliConfig(SimpleWikipediaConfig, CliConfig):
-    @staticmethod
-    def get_cli_options() -> t.List[click.Option]:
-        options = [
-            click.Option(
-                ["--page-title"],
-                required=True,
-                type=str,
-                help='Title of a Wikipedia page, e.g. "Open source software".',
-            ),
-            click.Option(
-                ["--auto-suggest"],
-                default=True,
-                is_flag=True,
-                help="Whether to automatically suggest a page if the exact page was not found."
-                " Set to False if the wrong Wikipedia page is fetched.",
-            ),
-        ]
-        return options
-
-
-def get_base_src_cmd() -> BaseSrcCmd:
-    cmd_cls = BaseSrcCmd(
-        cmd_name="wikipedia",
-        cli_config=WikipediaCliConfig,
-    )
-    return cmd_cls

unstructured_ingest/cli/common.py

@@ -1,7 +0,0 @@
-import logging
-
-from unstructured_ingest.logger import ingest_log_streaming_init
-
-
-def log_options(options: dict, verbose=False):
-    ingest_log_streaming_init(logging.DEBUG if verbose else logging.INFO)