validio-cli 0.24.1__tar.gz → 0.25.1__tar.gz

{validio_cli-0.24.1 → validio_cli-0.25.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validio-cli
-Version: 0.24.1
+Version: 0.25.1
 Summary: CLI tool to interact with the Validio platform
 Home-page: https://validio.io/
 License: Apache-2.0
@@ -16,7 +16,7 @@ Requires-Dist: classdiff (>=0.5.0,<0.6.0)
 Requires-Dist: prompt-toolkit (>=3.0.38,<4.0.0)
 Requires-Dist: pydantic (==2.4.2)
 Requires-Dist: tabulate (>=0.9.0,<0.10.0)
-Requires-Dist: typer[all] (>=0.7.0,<0.8.0)
+Requires-Dist: typer[all] (>=0.8.0,<0.9.0)
 Requires-Dist: types-tabulate (>=0.9.0.2,<0.10.0.0)
 Requires-Dist: validio-sdk
 Project-URL: Documentation, https://docs.validio.io/

{validio_cli-0.24.1 → validio_cli-0.25.1}/pyproject.toml

@@ -3,7 +3,7 @@ name = "validio-cli"
 # This version does not represent the released version or any tag. For each
 # release we automatically bump this before building and publishing so this
 # should be kept at 0.0.1dev1
-version = "0.24.1"
+version = "0.25.1"
 description = "CLI tool to interact with the Validio platform"
 authors = ["Validio <support@validio.io>"]
 license = "Apache-2.0"
@@ -16,7 +16,7 @@ readme = "README_PUBLIC.md"
 python = "^3.10"
 camel-converter = "^3.0.0"
 classdiff = "^0.5.0"
-typer = { extras = ["all"], version = "^0.7.0" }
+typer = { extras = ["all"], version = "^0.8.0" }
 prompt-toolkit = "^3.0.38"
 tabulate = "^0.9.0"
 types-tabulate = "^0.9.0.2"

{validio_cli-0.24.1 → validio_cli-0.25.1}/validio_cli/bin/entities/code.py

@@ -7,7 +7,6 @@ from typing import Any
 
 import classdiff  # type: ignore
 import typer
-from validio_sdk import ValidioError
 from validio_sdk._api.api import APIClient
 from validio_sdk.code import _import as code_import
 from validio_sdk.code import apply as code_apply
@@ -99,6 +98,15 @@ update_all_schemas_option = typer.Option(
     ),
 )
 
+show_progress_option = typer.Option(
+    True,
+    help=(
+        f"{BETA_BANNER} "
+        "If enabled, progress updates will be displayed during "
+        "execution of this command"
+    ),
+)
+
 
 def target_option(resource_type: str) -> Any:
     return typer.Option(
@@ -145,6 +153,7 @@ async def plan(
     source: list[str] = target_option("source"),
     validator: list[str] = target_option("validator"),
     window: list[str] = target_option("window"),
+    progress: bool = show_progress_option,
 ) -> None:
     dir_path = directory_or_default(directory)
     client, _ = get_client(config_dir)
@@ -174,6 +183,7 @@ async def plan(
             validators=set(validator),
             windows=set(window),
         ),
+        show_progress=progress,
     )
 
 
@@ -199,6 +209,7 @@ async def apply(
     source: list[str] = target_option("source"),
     validator: list[str] = target_option("validator"),
     window: list[str] = target_option("window"),
+    progress: bool = show_progress_option,
 ) -> None:
     dir_path = directory_or_default(directory)
     client, _ = get_client(config_dir)
@@ -208,7 +219,7 @@ async def apply(
     )
 
     namespace = get_namespace(dir_path)
-    diff, manifest_ctx = await _plan(
+    plan_result = await _plan(
         namespace=namespace,
         client=client,
         directory=dir_path,
@@ -228,8 +239,10 @@ async def apply(
             validators=set(validator),
             windows=set(window),
         ),
+        show_progress=progress,
     )
 
+    diff = plan_result.graph_diff
     if diff.num_operations() == 0:
         return
 
@@ -242,9 +255,10 @@ async def apply(
     await code_apply.apply(
         namespace=namespace,
         client=client,
-        ctx=manifest_ctx,
+        ctx=plan_result.diff_context,
         diff=diff,
         show_secrets=show_secrets,
+        show_progress=progress,
     )
 
     print(
@@ -302,11 +316,6 @@ async def _import(
         credentials=set(credential), channels=set(channel)
     )
 
-    if resource_names.is_empty() and import_namespace is None:
-        raise ValidioError(
-            "--import-namespace is required when targeting resources to move"
-        )
-
     dir_path = directory_or_default(directory)
     client, _ = get_client(config_dir)
     namespace = get_namespace(dir_path)
@@ -325,7 +334,7 @@ async def _import(
         if num_moved_resources > 0:
             print(f"Moved {num_moved_resources} resources")
 
-    diff, _ = await _plan(
+    plan_result = await _plan(
         namespace=namespace,
         client=client,
         directory=dir_path,
@@ -339,11 +348,14 @@ async def _import(
         import_mode=True,
     )
 
-    if diff.num_deletes() == 0:
+    if plan_result.graph_diff.num_deletes() == 0:
         _print_no_changes()
         return
 
-    s = await code_import._import(ctx=diff.to_delete)
+    s = await code_import._import(
+        ctx=plan_result.graph_diff.to_delete,
+        tags_ctx=plan_result.tags_context,
+    )
     output.write_text(s)
     print(f"Generated file {output.absolute()}")
 
@@ -371,7 +383,8 @@ async def _plan(
     show_diff: bool = True,
     targets: code_plan.ResourceNames = code_plan.ResourceNames(),
     import_mode: bool = False,
-) -> tuple[GraphDiff, DiffContext]:
+    show_progress: bool = True,
+) -> code_plan.PlanResult:
     plan_result = await code_plan.plan(
         namespace=namespace,
         client=client,
@@ -382,16 +395,15 @@ async def _plan(
         show_secrets=show_secrets,
         targets=targets,
         import_mode=import_mode,
+        show_progress=show_progress,
     )
 
-    diff, manifest_ctx = plan_result.graph_diff, plan_result.diff_context
-
     if show_diff:
-        if diff.num_operations() == 0:
+        if plan_result.graph_diff.num_operations() == 0:
             _print_no_changes()
         else:
             _show_resources_diff(
-                diff=diff,
+                diff=plan_result.graph_diff,
                 show_schema=show_schema,
                 show_secrets=show_secrets,
                 escape=sys.stdout.isatty() and color,
@@ -400,7 +412,7 @@ async def _plan(
 
     _show_deprecations(plan_result.deprecations)
 
-    return diff, manifest_ctx
+    return plan_result
 
 
 def _show_resources_diff(
@@ -522,7 +534,7 @@ def _show_update_resource_diff(
         if r.replacement_cascaded_update_parent:
             (cls, parent_name) = r.replacement_cascaded_update_parent
             cascade_update = (
-                f" (due to replacement of " f"{cls.__name__}({repr(parent_name)}))"
+                f" (due to replacement of " f"{cls.__name__}({parent_name!r}))"
             )
 
         print(f"{class_key} '{r.manifest.name}' will be updated{cascade_update}")

{validio_cli-0.24.1 → validio_cli-0.25.1}/validio_cli/bin/entities/sources.py

@@ -14,6 +14,7 @@ from validio_sdk._api.api import (
     SourceAction,
     apply_source_action,
     get_sources,
+    split_to_chunks,
 )
 from validio_sdk.client import Session
 from validio_sdk.config import ValidioConfig
@@ -36,8 +37,6 @@ from validio_cli import (
 from validio_cli.bin.entities import credentials
 from validio_cli.namespace import get_namespace
 
-NUM_CONCURRENT_CONNECTIONS = 5
-
 app = AsyncTyper(help="Data sources to ingest data from")
 
 infer_schema_app = AsyncTyper(help="Infer schema")
@@ -111,15 +110,16 @@ async def start(
     namespace: str = Namespace(),
     identifiers: list[str] = Identifiers,
 ) -> None:
-    chunks = split_to_chunks(identifiers, NUM_CONCURRENT_CONNECTIONS)
-    results = await asyncio.gather(
-        *[
-            apply_batch_source_action(config_dir, SourceAction.START, chunk, namespace)
-            for chunk in chunks
-        ]
-    )
-
-    return handle_source_action_response(results, SourceAction.START, output_format)
+    client, cfg = get_client(config_dir)
+    async with client.client as session:
+        await apply_source_action_on_chunks(
+            session,
+            cfg,
+            SourceAction.START,
+            split_to_chunks(identifiers),
+            namespace,
+            output_format,
+        )
 
 
 @app.async_command(help="Stop source")
@@ -130,15 +130,16 @@ async def stop(
     namespace: str = Namespace(),
     identifiers: list[str] = Identifiers,
 ) -> None:
-    chunks = split_to_chunks(identifiers, NUM_CONCURRENT_CONNECTIONS)
-    results = await asyncio.gather(
-        *[
-            apply_batch_source_action(config_dir, SourceAction.STOP, chunk, namespace)
-            for chunk in chunks
-        ]
-    )
-
-    return handle_source_action_response(results, SourceAction.STOP, output_format)
+    client, cfg = get_client(config_dir)
+    async with client.client as session:
+        await apply_source_action_on_chunks(
+            session,
+            cfg,
+            SourceAction.STOP,
+            split_to_chunks(identifiers),
+            namespace,
+            output_format,
+        )
 
 
 @app.async_command(help="Backfill source")
@@ -149,17 +150,16 @@ async def backfill(
     namespace: str = Namespace(),
     identifiers: list[str] = Identifiers,
 ) -> None:
-    chunks = split_to_chunks(identifiers, NUM_CONCURRENT_CONNECTIONS)
-    results = await asyncio.gather(
-        *[
-            apply_batch_source_action(
-                config_dir, SourceAction.BACKFILL, chunk, namespace
-            )
-            for chunk in chunks
-        ]
-    )
-
-    return handle_source_action_response(results, SourceAction.BACKFILL, output_format)
+    client, cfg = get_client(config_dir)
+    async with client.client as session:
+        await apply_source_action_on_chunks(
+            session,
+            cfg,
+            SourceAction.BACKFILL,
+            split_to_chunks(identifiers),
+            namespace,
+            output_format,
+        )
 
 
 @app.async_command(help="Reset source")
@@ -170,15 +170,16 @@ async def reset(
     namespace: str = Namespace(),
     identifiers: list[str] = Identifiers,
 ) -> None:
-    chunks = split_to_chunks(identifiers, NUM_CONCURRENT_CONNECTIONS)
-    results = await asyncio.gather(
-        *[
-            apply_batch_source_action(config_dir, SourceAction.RESET, chunk, namespace)
-            for chunk in chunks
-        ]
-    )
-
-    return handle_source_action_response(results, SourceAction.RESET, output_format)
+    client, cfg = get_client(config_dir)
+    async with client.client as session:
+        await apply_source_action_on_chunks(
+            session,
+            cfg,
+            SourceAction.RESET,
+            split_to_chunks(identifiers),
+            namespace,
+            output_format,
+        )
 
 
 @infer_schema_app.callback(invoke_without_command=True)
@@ -1145,8 +1146,11 @@ async def _get_null_marker() -> str | None:
     return null_marker
 
 
-async def get_source_id(
-    executor: APIClient | Session, cfg: ValidioConfig, identifier: str, namespace: str
+async def _get_source_id(
+    session: Session,
+    cfg: ValidioConfig,
+    identifier: str,
+    namespace: str,
 ) -> str | None:
     """
     Ensure the identifier is a resource id.
@@ -1164,7 +1168,7 @@ async def get_source_id(
         return identifier
 
     resource = await get_sources(
-        executor,
+        session,
         source_id=identifier,
         namespace_id=get_namespace(namespace, cfg),
     )
@@ -1179,6 +1183,16 @@ async def get_source_id(
     return resource["id"]
 
 
+async def get_source_id(
+    client: APIClient,
+    cfg: ValidioConfig,
+    identifier: str,
+    namespace: str,
+) -> str | None:
+    async with client.client as session:
+        return await _get_source_id(session, cfg, identifier, namespace)
+
+
 def _resource_filter(
     source: Any, credential_id: str | None, credential_name: str | None
 ) -> bool:
@@ -1234,15 +1248,35 @@ def _interactive_message_format_input() -> tuple[str, list[str], str]:
     )
 
 
+async def apply_source_action_on_chunks(
+    session: Session,
+    cfg: ValidioConfig,
+    action: SourceAction,
+    chunks: list[list[str]],
+    namespace: str,
+    output_format: OutputFormat = OutputFormatOption,
+) -> None:
+    results = await asyncio.gather(
+        *[
+            apply_batch_source_action(session, cfg, action, chunk, namespace)
+            for chunk in chunks
+        ]
+    )
+
+    return handle_source_action_response(results, action, output_format)
+
+
 async def apply_batch_source_action(
-    config_dir: str, action: SourceAction, identifiers: list[str], namespace: str
+    session: Session,
+    cfg: ValidioConfig,
+    action: SourceAction,
+    identifiers: list[str],
+    namespace: str,
 ) -> list[dict[str, Any]]:
-    client, cfg = get_client(config_dir)
-    async with client.client as session:
-        return [
-            await apply_single_source_action(session, action, cfg, id, namespace)
-            for id in identifiers
-        ]
+    return [
+        await apply_single_source_action(session, action, cfg, id, namespace)
+        for id in identifiers
+    ]
 
 
 async def apply_single_source_action(
@@ -1252,7 +1286,7 @@ async def apply_single_source_action(
     identifier: str,
     namespace: str,
 ) -> dict[str, Any]:
-    source_id = await get_source_id(session, cfg, identifier, namespace)
+    source_id = await _get_source_id(session, cfg, identifier, namespace)
     if source_id is None:
         return {
             f"source{action.value}": {
@@ -1301,9 +1335,5 @@ def handle_source_action_response(
     )
 
 
-def split_to_chunks(data: list[Any], num_of_chunks: int) -> list[list[Any]]:
-    return [data[i::num_of_chunks] for i in range(min(num_of_chunks, len(data)))]
-
-
 if __name__ == "__main__":
     typer.run(app())