validio-cli 0.29.0__tar.gz → 5.3.0__tar.gz

{validio_cli-0.29.0 → validio_cli-5.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-cli
-Version: 0.29.0
+Version: 5.3.0
 Summary: CLI tool to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0

{validio_cli-0.29.0 → validio_cli-5.3.0}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-cli"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "0.29.0"
+version = "5.3.0"
 description = "CLI tool to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"

{validio_cli-0.29.0 → validio_cli-5.3.0}/validio_cli/__init__.py

@@ -138,7 +138,7 @@ def output_json(obj: Any, identifier: str | None = None) -> None:
     print(j)
 
 
-def output_text(items: Any, fields: dict[str, OutputSettings | None]) -> None:
+def format_text(items: Any, fields: dict[str, OutputSettings | None]) -> str:
     if items is None:
         items = []
     elif not isinstance(items, list):
@@ -175,8 +175,11 @@ def output_text(items: Any, fields: dict[str, OutputSettings | None]) -> None:
 
         table.append(row)
 
-    datatable = tabulate(table, tablefmt="plain")
-    print(datatable)
+    return tabulate(table, tablefmt="plain")
+
+
+def output_text(items: Any, fields: dict[str, OutputSettings | None]) -> None:
+    print(format_text(items=items, fields=fields))
 
     # We flush stdout here so we can avoid an unhandled broken pipe error.
     # Catching the exception according to the documentation at

{validio_cli-0.29.0 → validio_cli-5.3.0}/validio_cli/bin/entities/sources.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import uuid
 from pathlib import Path
-from typing import Any, cast
+from typing import Any, Optional, cast
 
 import click
 import typer
@@ -30,12 +30,14 @@ from validio_cli import (
     OutputFormatOption,
     OutputSettings,
     components,
+    format_text,
     get_client,
     output_json,
     output_ok_or_error,
     output_text,
 )
 from validio_cli.bin.entities import credentials
+from validio_cli.components import proceed_with_operation
 from validio_cli.namespace import get_namespace
 
 app = AsyncTyper(help="Data sources to ingest data from")
@@ -132,25 +134,96 @@ async def delete(
     return output_ok_or_error(result)
 
 
+async def prompt_for_start_all_sources(
+    client: APIClient,
+    cfg: ValidioConfig,
+    namespace: str,
+    auto_approve: bool,
+) -> list[str] | None:
+    if not namespace:
+        raise typer.BadParameter("Namespace must be supplied when using --all")
+
+    sources = await client.get_sources(
+        namespace_id=get_namespace(namespace, cfg),
+    )
+
+    if not isinstance(sources, list):
+        raise ValidioError("failed to get sources")
+
+    if len(sources) == 0:
+        print("No sources exist in namespace to start")
+        return None
+
+    sources_to_start = [
+        source for source in sources if source.get("state") in ["IDLE", "INIT"]
+    ]
+
+    if len(sources_to_start) == 0:
+        print("No sources in namespace that need starting")
+        return None
+
+    prompt = "The following sources will be started:\n"
+    prompt += format_text(
+        sources_to_start,
+        fields={
+            "id": OutputSettings(attribute_name="id"),
+            "resource_name": OutputSettings(attribute_name="resourceName"),
+            "type": OutputSettings.trimmed_upper_snake(
+                attribute_name="__typename", trim="Source"
+            ),
+            "state": None,
+        },
+    )
+
+    print(prompt)
+
+    if not await proceed_with_operation(auto_approve):
+        return None
+    print()
+
+    return [source["id"] for source in sources_to_start]
+
+
 @app.async_command(help="Start sources")
 async def start(
     config_dir: str = ConfigDir,
     output_format: OutputFormat = OutputFormatOption,
     # ruff: noqa: ARG001
     namespace: str = Namespace(),
-    identifiers: list[str] = Identifiers,
+    all: bool = typer.Option(False, "--all", help="Start all unstarted sources"),
+    auto_approve: bool = typer.Option(
+        False,
+        help="When used with --all, automatically approve starting of all sources",
+    ),
+    identifiers: Optional[list[str]] = Identifiers,
 ) -> None:
     client, cfg = get_client(config_dir)
-    async with client.client as session:
-        await apply_source_action_on_chunks(
-            session,
-            cfg,
-            SourceAction.START,
-            split_to_chunks(identifiers),
-            namespace,
-            output_format,
+
+    if all and identifiers and len(identifiers) > 0:
+        raise typer.BadParameter(
+            "Cannot use both --all and supplying source "
+            "identifiers. You must only use one of them"
         )
 
+    if all:
+        identifiers = await prompt_for_start_all_sources(
+            client=client,
+            cfg=cfg,
+            namespace=namespace,
+            auto_approve=auto_approve,
+        )
+
+    if identifiers:
+        async with client.client as session:
+            await apply_source_action_on_chunks(
+                session,
+                cfg,
+                SourceAction.START,
+                split_to_chunks(identifiers),
+                namespace,
+                output_format,
+            )
+
 
 @app.async_command(help="Stop source")
 async def stop(
@@ -271,13 +344,11 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
     credential_type_to_source_type = {
         "AwsCredential": [
             ("kinesis", "Amazon Kinesis"),
-            ("s3", "Amazon S3"),
         ],
         "DatabricksCredential": [("databricks", "Databricks")],
         "DemoCredential": [("demo", "Demo")],
         "GcpCredential": [
             ("bigquery", "Google BigQuery"),
-            ("gcs", "Google Cloud Storage"),
             ("pubsub", "Google Pub/Sub"),
         ],
         "PostgreSqlCredential": [("postgresql", "PostgreSQL")],
@@ -458,29 +529,6 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
                 info.get("message_format", ""),
                 info.get("message_schema", ""),
             )
-        case "s3":
-            info = await _multip_prompt(
-                [
-                    ("Bucket", [], ""),
-                    ("File pattern", [], "*.csv"),
-                    ("Prefix", [], ""),
-                    ("CSV delimiter", [",", "|"], ","),
-                ]
-            )
-
-            print()
-            null_marker = await _get_null_marker()
-
-            await _infer_schema_s3(
-                vc,
-                filename,
-                credential_id,
-                info.get("bucket", ""),
-                info.get("file_pattern", ""),
-                info.get("prefix", ""),
-                info.get("csv_delimiter", ""),
-                null_marker,
-            )
         case "postgresql":
             info = await _multip_prompt(
                 [
@@ -549,31 +597,6 @@ async def _interactive(config_dir: str, filename: Path, namespace: str) -> None:
                 info.get("dataset", ""),
                 info.get("table", ""),
             )
-        case "gcs":
-            info = await _multip_prompt(
-                [
-                    ("Project", [], ""),
-                    ("Bucket", [], ""),
-                    ("Folder", [], ""),
-                    ("File pattern", ["*.csv"], "*.csv"),
-                    ("CSV delimiter", [",", "|"], ","),
-                ]
-            )
-
-            print()
-            null_marker = await _get_null_marker()
-
-            await _infer_schema_gcs(
-                vc,
-                filename,
-                credential_id,
-                info.get("project", ""),
-                info.get("bucket", ""),
-                info.get("folder", ""),
-                info.get("file_pattern", ""),
-                info.get("csv_delimiter", ""),
-                null_marker,
-            )
         case _:
             print("Not yet implemented...")
             return
@@ -588,40 +611,6 @@ async def demo(
     await _infer_schema_demo(vc, filename)
 
 
-@infer_schema_app.async_command(help="Infer Amazon S3 schema")
-async def s3(
-    config_dir: str = ConfigDir,
-    filename: Path = schema_filename_option("s3"),
-    namespace: str = Namespace(),
-    credential_id: str = typer.Option(..., help="Credential name or ID"),
-    bucket: str = typer.Option(..., help="S3 bucket name"),
-    file_pattern: str = typer.Option(
-        "*.csv", help="File glob pattern - files to use for inference"
-    ),
-    prefix: str = typer.Option("", help="Prefix in the bucket, a directory"),
-    csv_delimiter: str = typer.Option(",", help="Delimiter between columns in file"),
-    null_marker: str = typer.Option(None, help="Null marker (values to treat as NULL)"),
-) -> None:
-    vc, cfg = get_client(config_dir)
-
-    resolved_credential_id = await credentials.get_credential_id(
-        vc, cfg, credential_id, namespace
-    )
-    if resolved_credential_id is None:
-        return
-
-    await _infer_schema_s3(
-        vc,
-        filename,
-        resolved_credential_id,
-        bucket,
-        file_pattern,
-        prefix,
-        csv_delimiter,
-        null_marker,
-    )
-
-
 @infer_schema_app.async_command(help="Infer Amazon Kinesis schema")
 async def kinesis(
     config_dir: str = ConfigDir,
@@ -792,40 +781,6 @@ async def bigquery(
     )
 
 
-@infer_schema_app.async_command(help="Infer Google Cloud Storage schema")
-async def gcs(
-    config_dir: str = ConfigDir,
-    filename: Path = schema_filename_option("gcs"),
-    namespace: str = Namespace(),
-    credential_id: str = typer.Option(..., help="Credential name or ID"),
-    project: str = typer.Option(..., help="Google project name"),
-    bucket: str = typer.Option(..., help="GCS bucket"),
-    folder: str = typer.Option("", help="Folder in the bucket"),
-    file_pattern: str = typer.Option("*.csv", help="File glob"),
-    csv_delimiter: str = typer.Option(",", help="Delimiter between columns in file"),
-    null_marker: str = typer.Option(None, help="Null marker (values to treat as NULL)"),
-) -> None:
-    vc, cfg = get_client(config_dir)
-
-    resolved_credential_id = await credentials.get_credential_id(
-        vc, cfg, credential_id, namespace
-    )
-    if resolved_credential_id is None:
-        return
-
-    await _infer_schema_gcs(
-        vc,
-        filename,
-        resolved_credential_id,
-        project,
-        bucket,
-        folder,
-        file_pattern,
-        csv_delimiter,
-        null_marker,
-    )
-
-
 async def _infer_schema_databricks(
     vc: APIClient,
     filename: Path,
@@ -855,32 +810,6 @@ async def _infer_schema_demo(
     _write_schema(filename, schema)
 
 
-async def _infer_schema_s3(
-    vc: APIClient,
-    filename: Path,
-    credential_id: str,
-    bucket: str,
-    file_pattern: str,
-    prefix: str,
-    csv_delimiter: str,
-    null_marker: str | None,
-) -> None:
-    schema = await vc.infer_schema(
-        class_name="AwsS3",
-        variable_values={
-            "credentialId": credential_id,
-            "bucket": bucket,
-            "filePattern": file_pattern,
-            "prefix": prefix,
-            "csv": {
-                "delimiter": csv_delimiter,
-                "nullMarker": null_marker,
-            },
-        },
-    )
-    _write_schema(filename, schema)
-
-
 async def _infer_schema_kinesis(
     vc: APIClient,
     filename: Path,
@@ -1029,34 +958,6 @@ async def _infer_schema_bigquery(
     _write_schema(filename, schema)
 
 
-async def _infer_schema_gcs(
-    vc: APIClient,
-    filename: Path,
-    credential_id: str,
-    project: str,
-    bucket: str,
-    folder: str,
-    file_pattern: str,
-    csv_delimiter: str,
-    null_marker: str | None,
-) -> None:
-    schema = await vc.infer_schema(
-        class_name="GcpStorage",
-        variable_values={
-            "credentialId": credential_id,
-            "project": project,
-            "bucket": bucket,
-            "folder": folder,
-            "filePattern": file_pattern,
-            "csv": {
-                "delimiter": csv_delimiter,
-                "nullMarker": null_marker,
-            },
-        },
-    )
-    _write_schema(filename, schema)
-
-
 def _write_schema(filename: Path, schema: Any) -> None:
     filename.write_text(json.dumps(schema, indent=2))
 
@@ -1081,38 +982,6 @@ async def _multip_prompt(questions: list[tuple[str, list[str], str]]) -> dict[st
     return answers
 
 
-async def _get_null_marker() -> str | None:
-    null_marker = await components.radiolist_dialog(
-        title="Null marker - What value should be treated as NULL",
-        values=[
-            ("none", "None (No value is treated as NULL)"),
-            ("NULL", "The literal string 'NULL', f.ex. 'a,NULL,c'"),
-            ("empty", "Empty (\"\", f.ex 'a,,c')"),
-            ("other", "Set manually"),
-        ],
-    )
-
-    # User hit escape sequence, don't return None in this case.
-    if null_marker is None:
-        raise typer.Exit(code=1)
-
-    if null_marker == "other":
-        session: PromptSession = PromptSession()
-        null_marker = await session.prompt_async(
-            validio_cli._fixed_width("Null marker"),
-        )
-
-    if null_marker == "none":
-        return None
-
-    # We can't map the value to '""' because it will make it pre-selected in the
-    # dialog which we don't want.
-    if null_marker == "empty":
-        return ""
-
-    return null_marker
-
-
 async def _get_source_id(
     session: Session,
     cfg: ValidioConfig,
@@ -1227,7 +1096,11 @@ async def apply_source_action_on_chunks(
         ]
     )
 
-    return handle_source_action_response(results, action, output_format)
+    return await handle_source_action_response(
+        results,
+        action,
+        output_format,
+    )
 
 
 async def apply_batch_source_action(
@@ -1253,15 +1126,17 @@ async def apply_single_source_action(
     source_id = await _get_source_id(session, cfg, identifier, namespace)
     if source_id is None:
         return {
-            f"source{action.value}": {
-                "errors": [{"message": f"No source with id or name {identifier} found"}]
-            }
+            "errors": [{"message": f"No source with id or name {identifier} found"}]
         }
 
-    return await apply_source_action(session, action, source_id)
+    result = await apply_source_action(session, action, source_id)
+    return {
+        "id": source_id,
+        **result,
+    }
 
 
-def handle_source_action_response(
+async def handle_source_action_response(
     results: list[list[dict[str, Any]]],
     action: SourceAction,
     output_format: OutputFormat,
@@ -1290,7 +1165,19 @@ def handle_source_action_response(
     if output_format == OutputFormat.JSON:
         return output_json(result)
 
-    return output_text(
+    output_action_word = ""
+    match action:
+        case SourceAction.START:
+            output_action_word = "starting"
+        case SourceAction.STOP:
+            output_action_word = "stopping"
+        case SourceAction.BACKFILL:
+            output_action_word = "initiating backfilling"
+        case SourceAction.RESET:
+            output_action_word = "resetting"
+
+    print(f"Summary of results when {output_action_word} sources:")
+    output_text(
         result,
         fields={
             field_name: None,
@@ -1298,6 +1185,26 @@ def handle_source_action_response(
         },
     )
 
+    if len(error_responses) > 0:
+        print()
+        print(f"Failures details when {output_action_word} sources:")
+        output_text(
+            error_responses,
+            fields={
+                "source": OutputSettings(
+                    attribute_name="id",
+                ),
+                "error": OutputSettings(
+                    attribute_name="errors",
+                    reformat=lambda errors: ",".join(
+                        [error["message"] for error in errors]
+                    ),
+                ),
+            },
+        )
+
+    return None
+
 
 if __name__ == "__main__":
     typer.run(app())