tinybird 0.0.1.dev43__py3-none-any.whl → 0.0.1.dev46__py3-none-any.whl

tinybird/tb/modules/datafile/pipe_checker.py

@@ -288,8 +288,8 @@ class PipeCheckerRunner:
                             AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                             AND error = 0
                             AND not mapContains(parameters, '__tb__semver')
-                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
-                            { extra_where_clause }
+                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
+                            {extra_where_clause}
                         Limit 5000000 -- Enough to bring data while not processing all requests from highly used pipes
                         )
                     group by request_param_names, http_method
@@ -315,7 +315,7 @@ class PipeCheckerRunner:
                                         AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                                         AND error = 0
                                         AND not mapContains(parameters, '__tb__semver')
-                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
+                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
                                         {extra_where_clause}
                                     LIMIT {limit}
                                 )

tinybird/tb/modules/datasource.py

@@ -30,7 +30,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 @cli.group()
 @click.pass_context
 def datasource(ctx):
-    """Data Sources commands"""
+    """Data Source commands"""
 
 
 @datasource.command(name="ls")

tinybird/tb/modules/deployment.py

@@ -191,7 +191,7 @@ def deployment_rollback(ctx: click.Context) -> None:
     rollback_deployment(client.host, HEADERS)
 
 
-@cli.command(name="deploy")
+@cli.command(name="deploy", hidden=True)
 @click.option(
     "--wait/--no-wait",
     is_flag=True,

tinybird/tb/modules/endpoint.py

@@ -5,7 +5,7 @@
 
 import json
 import re
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 from urllib.parse import urlencode
 
 import click
@@ -21,7 +21,7 @@ from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-@cli.group(hidden=True)
+@cli.group()
 @click.pass_context
 def endpoint(ctx):
     """Endpoint commands"""
@@ -185,3 +185,90 @@ def get_endpoint_token(tokens: List[Dict[str, Any]], pipe_name: str) -> Optional
                 break
 
     return token
+
+
+@endpoint.command(name="stats")
+@click.argument("pipes", nargs=-1)
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Force a type of the output. To parse the output, keep in mind to use `tb --no-version-warning endpoint stats` option.",
+)
+@click.pass_context
+@coro
+async def endpoint_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: str):
+    """
+    Print endpoint stats for the last 7 days
+    """
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    all_pipes = await client.pipes()
+    pipes_to_get_stats = []
+    pipes_ids: Dict = {}
+
+    if pipes:
+        # We filter by the pipes we want to look for
+        all_pipes = [pipe for pipe in all_pipes if pipe["name"] in pipes]
+
+    for pipe in all_pipes:
+        name_version = get_name_version(pipe["name"])
+        if name_version["name"] in pipe["name"]:
+            pipes_to_get_stats.append(f"'{pipe['id']}'")
+            pipes_ids[pipe["id"]] = name_version
+
+    if not pipes_to_get_stats:
+        if format_ == "json":
+            click.echo(json.dumps({"pipes": []}, indent=2))
+        else:
+            click.echo(FeedbackManager.info_no_pipes_stats())
+        return
+
+    sql = f"""
+        SELECT
+            pipe_id id,
+            sumIf(view_count, date > now() - interval 7 day) requests,
+            sumIf(error_count, date > now() - interval 7 day) errors,
+            avgMergeIf(avg_duration_state, date > now() - interval 7 day) latency
+        FROM tinybird.pipe_stats
+        WHERE pipe_id in ({",".join(pipes_to_get_stats)})
+        GROUP BY pipe_id
+        ORDER BY requests DESC
+        FORMAT JSON
+    """
+
+    res = await client.query(sql)
+
+    if res and "error" in res:
+        raise CLIPipeException(FeedbackManager.error_exception(error=str(res["error"])))
+
+    columns = ["name", "request count", "error count", "avg latency"]
+    table_human_readable: List[Tuple] = []
+    table_machine_readable: List[Dict] = []
+    if res and "data" in res:
+        for x in res["data"]:
+            tk = pipes_ids[x["id"]]
+            table_human_readable.append(
+                (
+                    tk["name"],
+                    x["requests"],
+                    x["errors"],
+                    x["latency"],
+                )
+            )
+            table_machine_readable.append(
+                {
+                    "name": tk["name"],
+                    "requests": x["requests"],
+                    "errors": x["errors"],
+                    "latency": x["latency"],
+                }
+            )
+
+        table_human_readable.sort(key=lambda x: (x[1], x[0]))
+        table_machine_readable.sort(key=lambda x: x["name"])
+
+        if format_ == "json":
+            click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
+        else:
+            echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)

tinybird/tb/modules/feedback_manager.py

@@ -432,6 +432,7 @@ class FeedbackManager:
     error_updating_tag = error_message("Error updating tag: {error}")
     error_tag_generic = error_message("There was an issue updating tags. {error}")
     error_tag_not_found = error_message("Tag {tag_name} not found.")
+    error_build_failed = error_message("Build failed")
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(
@@ -849,6 +850,7 @@ Ready? """
     )
     info_tag_list = info_message("** Tags:")
     info_tag_resources = info_message("** Resources tagged by {tag_name}:")
+    info_build_failed = info_message("** Build failed")
     warning_no_release = warning_message(
         "** Warning: Workspace does not have Releases, run `tb init --git` to activate them."
     )
@@ -964,7 +966,7 @@ Ready? """
     )
     success_datasource_alter = success_message("** The Data Source has been correctly updated.")
     success_datasource_kafka_connected = success_message(
-        "** Data Source '{id}' created\n" "** Kafka streaming connection configured successfully!"
+        "** Data Source '{id}' created\n** Kafka streaming connection configured successfully!"
     )
     success_datasource_shared = success_message(
         "** The Data Source {datasource} has been correctly shared with {workspace}"
@@ -1040,6 +1042,8 @@ Ready? """
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)
 
+    highlight_building_project = info_highlight_message("\n» Building project...")
+
     success = success_message("{message}")
     info = info_message("{message}")
     highlight = info_highlight_message("{message}")

tinybird/tb/modules/local_common.py

@@ -1,7 +1,6 @@
 import hashlib
 import logging
 import os
-from typing import Optional
 
 import requests
 
@@ -9,19 +8,19 @@ from tinybird.client import TinyB
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.exceptions import CLIException
 
-TB_IMAGE_NAME = "tinybirdco/tinybird-local:beta"
+TB_IMAGE_NAME = "registry.gitlab.com/tinybird/analytics/tinybird-local-jammy-3.11:beta"
 TB_CONTAINER_NAME = "tinybird-local"
 TB_LOCAL_PORT = int(os.getenv("TB_LOCAL_PORT", 80))
 TB_LOCAL_HOST = f"http://localhost:{TB_LOCAL_PORT}"
 
 
-async def get_tinybird_local_client(path: Optional[str] = None) -> TinyB:
+async def get_tinybird_local_client(path: str, build: bool = False) -> TinyB:
     """Get a Tinybird client connected to the local environment."""
-    config = await get_tinybird_local_config(path)
+    config = await get_tinybird_local_config(path, build=build)
     return config.get_client(host=TB_LOCAL_HOST)
 
 
-async def get_tinybird_local_config(path: Optional[str] = None) -> CLIConfig:
+async def get_tinybird_local_config(path: str, build: bool = False) -> CLIConfig:
     """Craft a client config with a workspace name based on the path of the project files
 
     It uses the tokens from tinybird local
@@ -41,11 +40,15 @@ async def get_tinybird_local_config(path: Optional[str] = None) -> CLIConfig:
     if path:
         folder_hash = hashlib.sha256(path.encode()).hexdigest()
         user_client = config.get_client(host=TB_LOCAL_HOST, token=user_token)
-        ws_name = f"Tinybird_Local_Build_{folder_hash}"
+        ws_name = f"Tinybird_Local_Build_{folder_hash}" if build else f"Tinybird_Local_{folder_hash}"
         logging.debug(f"Workspace used for build: {ws_name}")
 
         user_workspaces = requests.get(f"{TB_LOCAL_HOST}/v0/user/workspaces?token={user_token}").json()
-        local_workspaces = [ws for ws in user_workspaces["workspaces"] if ws["name"].startswith(ws_name)]
+        local_workspaces = (
+            [ws for ws in user_workspaces["workspaces"] if ws["name"].startswith(ws_name)]
+            if user_workspaces.get("workspaces")
+            else []
+        )
         local_workspaces = sorted(local_workspaces, key=lambda x: x["name"])
 
         ws = None

tinybird/tb/modules/materialization.py

@@ -0,0 +1,146 @@
+import json
+import re
+
+import click
+
+from tinybird.client import TinyB
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.common import (
+    coro,
+    create_tb_client,
+    echo_safe_humanfriendly_tables_format_smart_table,
+    wait_job,
+)
+from tinybird.tb.modules.datafile.common import PipeTypes, get_name_version
+from tinybird.tb.modules.exceptions import CLIPipeException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+@cli.group()
+@click.pass_context
+def materialization(ctx):
+    """Materialization commands"""
+
+
+@materialization.command(name="ls")
+@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["json"], case_sensitive=False),
+    default=None,
+    help="Force a type of the output",
+)
+@click.pass_context
+@coro
+async def materialization_ls(ctx: click.Context, match: str, format_: str):
+    """List materializations"""
+
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    pipes = await client.pipes(dependencies=True, node_attrs="name,materialized", attrs="name,updated_at,endpoint,type")
+    materializations = [p for p in pipes if p.get("type") == PipeTypes.MATERIALIZED]
+    materializations = sorted(materializations, key=lambda p: p["updated_at"])
+    datasources = await client.datasources()
+    columns = ["name", "updated at", "nodes", "target datasource"]
+    table_human_readable = []
+    table_machine_readable = []
+    pattern = re.compile(match) if match else None
+    for t in materializations:
+        tk = get_name_version(t["name"])
+        if pattern and not pattern.search(tk["name"]):
+            continue
+        target_datasource_id = next((n["materialized"] for n in t["nodes"] if n.get("materialized")), None)
+        target_datasource = next((d for d in datasources if d["id"] == target_datasource_id), None)
+        target_datasource_name = target_datasource.get("name", "") if target_datasource else ""
+        table_human_readable.append((tk["name"], t["updated_at"][:-7], len(t["nodes"]), target_datasource_name))
+        table_machine_readable.append(
+            {
+                "name": tk["name"],
+                "updated at": t["updated_at"][:-7],
+                "nodes": len(t["nodes"]),
+                "target datasource": target_datasource_name,
+            }
+        )
+
+    if not format_:
+        click.echo(FeedbackManager.info_pipes())
+        echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
+        click.echo("\n")
+    elif format_ == "json":
+        click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
+    else:
+        raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
+
+
+@materialization.command(name="populate")
+@click.argument("pipe_name")
+@click.option("--node", type=str, help="Name of the materialized node.", default=None, required=False)
+@click.option(
+    "--sql-condition",
+    type=str,
+    default=None,
+    help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
+)
+@click.option(
+    "--truncate", is_flag=True, default=False, help="Truncates the materialized Data Source before populating it."
+)
+@click.option(
+    "--unlink-on-populate-error",
+    is_flag=True,
+    default=False,
+    help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
+)
+@click.option(
+    "--wait",
+    is_flag=True,
+    default=False,
+    help="Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
+)
+@click.pass_context
+@coro
+async def pipe_populate(
+    ctx: click.Context,
+    pipe_name: str,
+    node: str,
+    sql_condition: str,
+    truncate: bool,
+    unlink_on_populate_error: bool,
+    wait: bool,
+):
+    """Populate the result of a Materialized Node into the target Materialized View"""
+    cl = create_tb_client(ctx)
+
+    pipe = await cl.pipe(pipe_name)
+
+    if pipe["type"] != PipeTypes.MATERIALIZED:
+        raise CLIPipeException(FeedbackManager.error_pipe_not_materialized(pipe=pipe_name))
+
+    if not node:
+        materialized_ids = [pipe_node["id"] for pipe_node in pipe["nodes"] if pipe_node.get("materialized") is not None]
+
+        if not materialized_ids:
+            raise CLIPipeException(FeedbackManager.error_populate_no_materialized_in_pipe(pipe=pipe_name))
+
+        elif len(materialized_ids) > 1:
+            raise CLIPipeException(FeedbackManager.error_populate_several_materialized_in_pipe(pipe=pipe_name))
+
+        node = materialized_ids[0]
+
+    response = await cl.populate_node(
+        pipe_name,
+        node,
+        populate_condition=sql_condition,
+        truncate=truncate,
+        unlink_on_populate_error=unlink_on_populate_error,
+    )
+    if "job" not in response:
+        raise CLIPipeException(response)
+
+    job_id = response["job"]["id"]
+    job_url = response["job"]["job_url"]
+    if sql_condition:
+        click.echo(FeedbackManager.info_populate_condition_job_url(url=job_url, populate_condition=sql_condition))
+    else:
+        click.echo(FeedbackManager.info_populate_job_url(url=job_url))
+    if wait:
+        await wait_job(cl, job_id, job_url, "Populating")

tinybird/tb/modules/mock.py

@@ -1,6 +1,7 @@
 import logging
 import os
 from pathlib import Path
+from typing import Optional
 
 import click
 
@@ -13,6 +14,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml
 from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.project import Project
 
 
 @cli.command()
@@ -21,24 +23,27 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
 @click.option(
     "--prompt",
     type=str,
-    default="Use the datasource schema to generate sample data",
+    default="",
     help="Extra context to use for data generation",
 )
-@click.option("--folder", type=str, default=os.getcwd(), help="Folder where datafiles will be placed")
+@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated data")
+@click.pass_context
 @coro
-async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
-    """Load sample data into a Data Source.
+async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, skip: bool) -> None:
+    """Generate sample data for a datasource.
 
     Args:
         datasource: Path to the datasource file to load sample data into
         rows: Number of events to send
         prompt: Extra context to use for data generation
-        folder: Folder where datafiles will be placed
+        skip: Skip following up on the generated data
     """
 
     try:
+        project: Project = ctx.ensure_object(dict)["project"]
         datasource_path = Path(datasource)
         datasource_name = datasource
+        folder = project.folder
         click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
         if datasource_path.suffix == ".datasource":
             datasource_name = datasource_path.stem
@@ -46,6 +51,9 @@ async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
             datasource_path = Path("datasources", f"{datasource}.datasource")
         datasource_path = Path(folder) / datasource_path
 
+        if not datasource_path.exists():
+            raise CLIException(f"Datasource '{datasource_path.stem}' not found")
+
         prompt_path = Path(folder) / "fixtures" / f"{datasource_name}.prompt"
         if not prompt or prompt == "Use the datasource schema to generate sample data":
             # load the prompt from the fixture.prompt file if it exists
@@ -68,17 +76,49 @@ async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
             click.echo(FeedbackManager.error(message="This action requires authentication. Run 'tb login' first."))
             return
         llm = LLM(user_token=user_token, host=user_client.host)
-        tb_client = await get_tinybird_local_client(os.path.abspath(folder))
+        tb_client = await get_tinybird_local_client(folder)
         prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
-        response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
-        sql = extract_xml(response, "sql")
-        if os.environ.get("TB_DEBUG", "") != "":
-            logging.debug(sql)
-        result = await tb_client.query(f"{sql} FORMAT JSON")
-        data = result.get("data", [])[:rows]
-        fixture_name = build_fixture_name(datasource_path.absolute().as_posix(), datasource_name, datasource_content)
-        persist_fixture(fixture_name, data, folder)
-        click.echo(FeedbackManager.success(message=f"✓ /fixtures/{fixture_name}.ndjson created with {rows} rows"))
+        iterations = 0
+        history = ""
+        fixture_path: Optional[Path] = None
+        sql = ""
+        while iterations < 10:
+            feedback = ""
+            if iterations > 0:
+                feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
+            if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
+                break
+            else:
+                if iterations > 0:
+                    if fixture_path:
+                        fixture_path.unlink()
+                    fixture_path = None
+                    click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
+
+                response = llm.ask(system_prompt=mock_prompt(rows, feedback, history), prompt=prompt)
+                sql = extract_xml(response, "sql")
+                result = await tb_client.query(f"{sql} FORMAT JSON")
+                data = result.get("data", [])[:rows]
+                fixture_name = build_fixture_name(str(datasource_path), datasource_name, datasource_content)
+                fixture_path = persist_fixture(fixture_name, data, folder)
+                click.echo(FeedbackManager.info(message=f"✓ /fixtures/{fixture_name}.ndjson created"))
+
+                if os.environ.get("TB_DEBUG", "") != "":
+                    logging.debug(sql)
+
+                history = (
+                    history
+                    + f"""
+                <result_iteration_{iterations}>
+                    {response}
+                </result_iteration_{iterations}>
+                """
+                )
+                if skip:
+                    break
+                iterations += 1
+
+        click.echo(FeedbackManager.success(message=f"✓ Sample data for {datasource_name} created with {rows} rows"))
 
     except Exception as e:
-        raise CLIException(FeedbackManager.error_exception(error=e))
+        click.echo(FeedbackManager.error_exception(error=f"Error: {e}"))