tinybird 0.0.1.dev43__py3-none-any.whl → 0.0.1.dev44__py3-none-any.whl

tinybird/tb/modules/datafile/build.py

@@ -15,14 +15,12 @@ from tinybird.client import TinyB
 from tinybird.sql import parse_table_structure, schema_to_sql_columns
 from tinybird.sql_template import get_used_tables_in_template, render_sql_template
 from tinybird.tb.modules.common import get_ca_pem_content
-from tinybird.tb.modules.datafile.build_common import update_tags_in_resource
-from tinybird.tb.modules.datafile.build_datasource import is_datasource, new_ds
+from tinybird.tb.modules.datafile.build_datasource import is_datasource
 from tinybird.tb.modules.datafile.build_pipe import (
     get_target_materialized_data_source_name,
     is_endpoint,
     is_endpoint_with_no_dependencies,
     is_materialized,
-    new_pipe,
 )
 from tinybird.tb.modules.datafile.common import (
     DEFAULT_CRON_PERIOD,
@@ -44,95 +42,32 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import get_tinybird_local_config
+from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.project import Project
 
 
 async def folder_build(
-    tb_client: TinyB,
+    project: Project,
     filenames: Optional[List[str]] = None,
-    folder: str = ".",
-    ignore_sql_errors: bool = False,
     is_internal: bool = False,
     is_vendor: bool = False,
     current_ws: Optional[Dict[str, Any]] = None,
     local_ws: Optional[Dict[str, Any]] = None,
     watch: bool = False,
 ):
-    config = await get_tinybird_local_config(folder)
     build = True
     dry_run = False
     force = True
     only_changes = True
     debug = False
-    check = True
-    populate = False
-    populate_subset = None
-    populate_condition = None
-    tests_to_run = 0
-    override_datasource = False
-    skip_confirmation = True
-    wait = False
-    unlink_on_populate_error = False
-    only_response_times = False
     run_tests = False
     verbose = False
-    as_standard = False
     raise_on_exists = False
     fork_downstream = True
     fork = False
     release_created = False
-    tests_relative_change = 0.01
-    tests_sample_by_params = 0
-    tests_filter_by = None
-    tests_failfast = False
-    tests_ignore_order = False
-    tests_validate_processed_bytes = False
-    tests_check_requests_from_branch = False
-    vendor_paths = []
-
-    vendor_path = Path("vendor")
-    user_token = config.get_user_token()
-    user_client = deepcopy(tb_client)
-
-    if user_token:
-        user_client.token = user_token
-
-    vendor_workspaces = []
-
-    if vendor_path.exists() and not is_vendor and not watch:
-        user_workspaces = await user_client.user_workspaces()
-        for x in vendor_path.iterdir():
-            if x.is_dir() and x.name:
-                if user_token:
-                    try:
-                        ws_to_delete = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == x.name), None)
-                        if ws_to_delete:
-                            await user_client.delete_workspace(ws_to_delete["id"], hard_delete_confirmation=x.name)
-                    except Exception:
-                        pass
-                    vendor_ws = await user_client.create_workspace(x.name, template=None)
-                    vendor_workspaces.append(vendor_ws)
-                vendor_paths.append((x.name, str(x)))
-
-    workspaces: List[Dict[str, Any]] = (await user_client.user_workspaces()).get("workspaces", [])
-
-    if not is_vendor:
-        local_workspace = await tb_client.workspace_info()
-        local_ws_id = local_workspace.get("id")
-        local_ws = next((ws for ws in workspaces if ws["id"] == local_ws_id), {})
-
-    current_ws = current_ws or local_ws
-
-    for vendor_ws in [ws for ws in workspaces if ws["name"] in [ws["name"] for ws in vendor_workspaces]]:
-        ws_client = deepcopy(tb_client)
-        ws_client.token = vendor_ws["token"]
-        shared_ws_path = Path(folder) / "vendor" / vendor_ws["name"]
-
-        if shared_ws_path.exists() and not is_vendor and not watch:
-            await folder_build(
-                ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws, local_ws=local_ws
-            )
-
+    folder = str(project.path)
+    tb_client = await get_tinybird_local_client(folder)
     datasources: List[Dict[str, Any]] = await tb_client.datasources()
     pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
 
@@ -150,7 +85,7 @@ async def folder_build(
         dir_path=folder,
         process_dependencies=True,
         skip_connectors=True,
-        vendor_paths=vendor_paths,
+        vendor_paths=[],
         current_ws=current_ws,
         only_changes=only_changes,
         fork_downstream=fork_downstream,
@@ -191,63 +126,14 @@ async def folder_build(
             resource = to_run[name]["resource"]
             if not dry_run:
                 if should_push_file(name, remote_resource_names, force, run_tests):
-                    click.echo(FeedbackManager.info_processing_new_resource(name=name, version=""))
-                    try:
-                        await exec_file(
-                            to_run[name],
-                            tb_client,
-                            force,
-                            check,
-                            debug and verbose,
-                            populate,
-                            populate_subset,
-                            populate_condition,
-                            unlink_on_populate_error,
-                            wait,
-                            user_token,
-                            override_datasource,
-                            ignore_sql_errors,
-                            skip_confirmation,
-                            only_response_times,
-                            run_tests,
-                            as_standard,
-                            tests_to_run,
-                            tests_relative_change,
-                            tests_sample_by_params,
-                            tests_filter_by,
-                            tests_failfast,
-                            tests_ignore_order,
-                            tests_validate_processed_bytes,
-                            tests_check_requests_from_branch,
-                            current_ws,
-                            local_ws,
-                            fork_downstream,
-                            fork,
-                            build,
-                            is_vendor,
-                        )
-                        if not run_tests:
-                            click.echo(
-                                FeedbackManager.success_create(
-                                    name=(
-                                        name
-                                        if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
-                                    )
-                                )
-                            )
-                    except Exception as e:
-                        filename = to_run[name]["filename"]
-                        exception = FeedbackManager.error_push_file_exception(
-                            filename=filename,
-                            error=e,
-                        )
-                        raise click.ClickException(exception)
+                    filename = to_run[name]["filename"]
+                    filename = filename.replace(f"{folder}/", "")
+                    click.echo(FeedbackManager.info(message=f"✓ {filename}"))
                 else:
                     if raise_on_exists:
                         raise AlreadyExistsException(
                             FeedbackManager.warning_name_already_exists(
-                                name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
+                                name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
                             )
                         )
                     else:
@@ -262,7 +148,7 @@ async def folder_build(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -304,19 +190,6 @@ async def folder_build(
         # We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
         groups = [group for group in toposort(dependencies_graph_fork_downstream)]
 
-        for group in groups:
-            for name in group:
-                is_vendor = resources_to_run_fork_downstream.get(name, {}).get("filename", "").startswith("vendor/")
-                if not is_vendor:
-                    try:
-                        await tb_client.datasource_delete(name, force=True)
-                    except Exception:
-                        pass
-                    try:
-                        await tb_client.pipe_delete(name)
-                    except Exception:
-                        pass
-
         groups.reverse()
         for group in groups:
             for name in group:
@@ -433,88 +306,6 @@ async def name_matches_existing_resource(resource: str, name: str, tb_client: Ti
     return False
 
 
-async def exec_file(
-    r: Dict[str, Any],
-    tb_client: TinyB,
-    force: bool,
-    check: bool,
-    debug: bool,
-    populate: bool,
-    populate_subset,
-    populate_condition,
-    unlink_on_populate_error,
-    wait_populate,
-    user_token: Optional[str],
-    override_datasource: bool = False,
-    ignore_sql_errors: bool = False,
-    skip_confirmation: bool = False,
-    only_response_times: bool = False,
-    run_tests=False,
-    as_standard=False,
-    tests_to_run: int = 0,
-    tests_relative_change: float = 0.01,
-    tests_to_sample_by_params: int = 0,
-    tests_filter_by: Optional[List[str]] = None,
-    tests_failfast: bool = False,
-    tests_ignore_order: bool = False,
-    tests_validate_processed_bytes: bool = False,
-    tests_check_requests_from_branch: bool = False,
-    current_ws: Optional[Dict[str, Any]] = None,
-    local_ws: Optional[Dict[str, Any]] = None,
-    fork_downstream: Optional[bool] = False,
-    fork: Optional[bool] = False,
-    build: Optional[bool] = False,
-    is_vendor: Optional[bool] = False,
-):
-    if debug:
-        click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
-    if r["resource"] == "pipes":
-        await new_pipe(
-            r,
-            tb_client,
-            force,
-            check,
-            populate,
-            populate_subset,
-            populate_condition,
-            unlink_on_populate_error,
-            wait_populate,
-            ignore_sql_errors=ignore_sql_errors,
-            only_response_times=only_response_times,
-            run_tests=run_tests,
-            as_standard=as_standard,
-            tests_to_run=tests_to_run,
-            tests_relative_change=tests_relative_change,
-            tests_to_sample_by_params=tests_to_sample_by_params,
-            tests_filter_by=tests_filter_by,
-            tests_failfast=tests_failfast,
-            tests_ignore_order=tests_ignore_order,
-            tests_validate_processed_bytes=tests_validate_processed_bytes,
-            override_datasource=override_datasource,
-            tests_check_requests_from_branch=tests_check_requests_from_branch,
-            fork_downstream=fork_downstream,
-            fork=fork,
-        )
-        await update_tags_in_resource(r, "pipe", tb_client)
-    elif r["resource"] == "datasources":
-        await new_ds(
-            r,
-            tb_client,
-            user_token,
-            force,
-            skip_confirmation=skip_confirmation,
-            current_ws=current_ws,
-            local_ws=local_ws,
-            fork_downstream=fork_downstream,
-            fork=fork,
-            build=build,
-            is_vendor=is_vendor,
-        )
-        await update_tags_in_resource(r, "datasource", tb_client)
-    else:
-        raise click.ClickException(FeedbackManager.error_unknown_resource(resource=r["resource"]))
-
-
 def get_remote_resource_name_without_version(remote_resource_name: str) -> str:
     """
     >>> get_remote_resource_name_without_version("r__datasource")

tinybird/tb/modules/datasource.py

@@ -30,7 +30,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 @cli.group()
 @click.pass_context
 def datasource(ctx):
-    """Data Sources commands"""
+    """Data Source commands"""
 
 
 @datasource.command(name="ls")

tinybird/tb/modules/deployment.py

@@ -191,7 +191,7 @@ def deployment_rollback(ctx: click.Context) -> None:
     rollback_deployment(client.host, HEADERS)
 
 
-@cli.command(name="deploy")
+@cli.command(name="deploy", hidden=True)
 @click.option(
     "--wait/--no-wait",
     is_flag=True,

tinybird/tb/modules/endpoint.py

@@ -5,7 +5,7 @@
 
 import json
 import re
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urlencode
 
 import click
@@ -21,7 +21,7 @@ from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-@cli.group(hidden=True)
+@cli.group()
 @click.pass_context
 def endpoint(ctx):
     """Endpoint commands"""
@@ -185,3 +185,90 @@ def get_endpoint_token(tokens: List[Dict[str, Any]], pipe_name: str) -> Optional
                 break
 
     return token
+
+
+@endpoint.command(name="stats")
+@click.argument("pipes", nargs=-1)
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Force a type of the output. To parse the output, keep in mind to use `tb --no-version-warning endpoint stats` option.",
+)
+@click.pass_context
+@coro
+async def endpoint_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: str):
+    """
+    Print endpoint stats for the last 7 days
+    """
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    all_pipes = await client.pipes()
+    pipes_to_get_stats = []
+    pipes_ids: Dict = {}
+
+    if pipes:
+        # We filter by the pipes we want to look for
+        all_pipes = [pipe for pipe in all_pipes if pipe["name"] in pipes]
+
+    for pipe in all_pipes:
+        name_version = get_name_version(pipe["name"])
+        if name_version["name"] in pipe["name"]:
+            pipes_to_get_stats.append(f"'{pipe['id']}'")
+            pipes_ids[pipe["id"]] = name_version
+
+    if not pipes_to_get_stats:
+        if format_ == "json":
+            click.echo(json.dumps({"pipes": []}, indent=2))
+        else:
+            click.echo(FeedbackManager.info_no_pipes_stats())
+        return
+
+    sql = f"""
+        SELECT
+            pipe_id id,
+            sumIf(view_count, date > now() - interval 7 day) requests,
+            sumIf(error_count, date > now() - interval 7 day) errors,
+            avgMergeIf(avg_duration_state, date > now() - interval 7 day) latency
+        FROM tinybird.pipe_stats
+        WHERE pipe_id in ({",".join(pipes_to_get_stats)})
+        GROUP BY pipe_id
+        ORDER BY requests DESC
+        FORMAT JSON
+    """
+
+    res = await client.query(sql)
+
+    if res and "error" in res:
+        raise CLIPipeException(FeedbackManager.error_exception(error=str(res["error"])))
+
+    columns = ["name", "request count", "error count", "avg latency"]
+    table_human_readable: List[Tuple] = []
+    table_machine_readable: List[Dict] = []
+    if res and "data" in res:
+        for x in res["data"]:
+            tk = pipes_ids[x["id"]]
+            table_human_readable.append(
+                (
+                    tk["name"],
+                    x["requests"],
+                    x["errors"],
+                    x["latency"],
+                )
+            )
+            table_machine_readable.append(
+                {
+                    "name": tk["name"],
+                    "requests": x["requests"],
+                    "errors": x["errors"],
+                    "latency": x["latency"],
+                }
+            )
+
+        table_human_readable.sort(key=lambda x: (x[1], x[0]))
+        table_machine_readable.sort(key=lambda x: x["name"])
+
+        if format_ == "json":
+            click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
+        else:
+            echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)

tinybird/tb/modules/materialization.py

@@ -0,0 +1,146 @@
+import json
+import re
+
+import click
+
+from tinybird.client import TinyB
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
+    coro,
+    create_tb_client,
+    echo_safe_humanfriendly_tables_format_smart_table,
+    wait_job,
+)
+from tinybird.tb.modules.datafile.common import PipeTypes, get_name_version
+from tinybird.tb.modules.exceptions import CLIPipeException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+@cli.group()
+@click.pass_context
+def materialization(ctx):
+    """Materialization commands"""
+
+
+@materialization.command(name="ls")
+@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Force a type of the output",
+)
+@click.pass_context
+@coro
+async def materialization_ls(ctx: click.Context, match: str, format_: str):
+    """List materializations"""
+
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    pipes = await client.pipes(dependencies=True, node_attrs="name,materialized", attrs="name,updated_at,endpoint,type")
+    materializations = [p for p in pipes if p.get("type") == PipeTypes.MATERIALIZED]
+    materializations = sorted(materializations, key=lambda p: p["updated_at"])
+    datasources = await client.datasources()
+    columns = ["name", "updated at", "nodes", "target datasource"]
+    table_human_readable = []
+    table_machine_readable = []
+    pattern = re.compile(match) if match else None
+    for t in materializations:
+        tk = get_name_version(t["name"])
+        if pattern and not pattern.search(tk["name"]):
+            continue
+        target_datasource_id = next((n["materialized"] for n in t["nodes"] if n.get("materialized")), None)
+        target_datasource = next((d for d in datasources if d["id"] == target_datasource_id), None)
+        target_datasource_name = target_datasource.get("name", "") if target_datasource else ""
+        table_human_readable.append((tk["name"], t["updated_at"][:-7], len(t["nodes"]), target_datasource_name))
+        table_machine_readable.append(
+            {
+                "name": tk["name"],
+                "updated at": t["updated_at"][:-7],
+                "nodes": len(t["nodes"]),
+                "target datasource": target_datasource_name,
+            }
+        )
+
+    if not format_:
+        click.echo(FeedbackManager.info_pipes())
+        echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
+        click.echo("\n")
+    elif format_ == "json":
+        click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
+    else:
+        raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
+
+
+@materialization.command(name="populate")
+@click.argument("pipe_name")
+@click.option("--node", type=str, help="Name of the materialized node.", default=None, required=False)
+@click.option(
+    "--sql-condition",
+    type=str,
+    default=None,
+    help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
+)
+@click.option(
+    "--truncate", is_flag=True, default=False, help="Truncates the materialized Data Source before populating it."
+)
+@click.option(
+    "--unlink-on-populate-error",
+    is_flag=True,
+    default=False,
+    help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
+)
+@click.option(
+    "--wait",
+    is_flag=True,
+    default=False,
+    help="Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
+)
+@click.pass_context
+@coro
+async def pipe_populate(
+    ctx: click.Context,
+    pipe_name: str,
+    node: str,
+    sql_condition: str,
+    truncate: bool,
+    unlink_on_populate_error: bool,
+    wait: bool,
+):
+    """Populate the result of a Materialized Node into the target Materialized View"""
+    cl = create_tb_client(ctx)
+
+    pipe = await cl.pipe(pipe_name)
+
+    if pipe["type"] != PipeTypes.MATERIALIZED:
+        raise CLIPipeException(FeedbackManager.error_pipe_not_materialized(pipe=pipe_name))
+
+    if not node:
+        materialized_ids = [pipe_node["id"] for pipe_node in pipe["nodes"] if pipe_node.get("materialized") is not None]
+
+        if not materialized_ids:
+            raise CLIPipeException(FeedbackManager.error_populate_no_materialized_in_pipe(pipe=pipe_name))
+
+        elif len(materialized_ids) > 1:
+            raise CLIPipeException(FeedbackManager.error_populate_several_materialized_in_pipe(pipe=pipe_name))
+
+        node = materialized_ids[0]
+
+    response = await cl.populate_node(
+        pipe_name,
+        node,
+        populate_condition=sql_condition,
+        truncate=truncate,
+        unlink_on_populate_error=unlink_on_populate_error,
+    )
+    if "job" not in response:
+        raise CLIPipeException(response)
+
+    job_id = response["job"]["id"]
+    job_url = response["job"]["job_url"]
+    if sql_condition:
+        click.echo(FeedbackManager.info_populate_condition_job_url(url=job_url, populate_condition=sql_condition))
+    else:
+        click.echo(FeedbackManager.info_populate_job_url(url=job_url))
+    if wait:
+        await wait_job(cl, job_id, job_url, "Populating")