tinybird 0.0.1.dev72__py3-none-any.whl → 0.0.1.dev74__py3-none-any.whl

tinybird/client.py

@@ -669,10 +669,12 @@ class TinyB:
     async def pipe_unlink_materialized(self, pipe_name: str, node_id: str):
         return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_id}/materialization", method="DELETE")
 
-    async def query(self, sql: str, pipeline: Optional[str] = None):
+    async def query(self, sql: str, pipeline: Optional[str] = None, playground: Optional[str] = None):
         params = {}
         if pipeline:
             params = {"pipeline": pipeline}
+        if playground:
+            params = {"playground": playground}
         params.update({"release_replacements": "true"})
 
         if len(sql) > TinyB.MAX_GET_LENGTH:

tinybird/prompts.py

@@ -634,20 +634,42 @@ materialized_pipe_instructions = """
 <materialized_pipe_instructions>
 - Do not create materialized pipes by default, unless the user asks for it.
 - In a .pipe file you can define how to materialize each row ingested in the earliest Data Source in the Pipe query to a materialized Data Source. Materialization happens at ingest.
-- DATASOURCE: Required when TYPE is MATERIALIZED. Sets the destination Data Source for materialized nodes.
+- DATASOURCE: Required when TYPE is MATERIALIZED. Sets the target Data Source for materialized nodes.
 - TYPE MATERIALIZED is the type of the pipe and it is mandatory for materialized pipes.
-- The content of the .pipe file must follow this format:
-DESCRIPTION Materialized Pipe to aggregate sales per hour in the sales_by_hour Data Source
-
+- The content of the .pipe file must follow the materialized_pipe_content format.
+- Use State modifier for the aggregated columns in the pipe.
+- Keep the SQL query simple and avoid using complex queries with joins, subqueries, etc.
+</materialized_pipe_instructions>
+<materialized_pipe_content>
 NODE daily_sales
 SQL >
-    SELECT toStartOfDay(starting_date) day, country, sum(sales) as total_sales
+    SELECT toStartOfDay(starting_date) day, country, sumState(sales) as total_sales
     FROM teams
     GROUP BY day, country
 
 TYPE MATERIALIZED
 DATASOURCE sales_by_hour
-</materialized_pipe_instructions>
+</materialized_pipe_content>
+<target_datasource_instructions>
+- The target datasource of a materialized pipe must have an AggregatingMergeTree engine.
+- Use AggregateFunction for the aggregated columns in the pipe.
+- Pipes using a materialized data source must use the Merge modifier in the SQL query for the aggregated columns. Example: sumMerge(total_sales)
+- Put all dimensions in the ENGINE_SORTING_KEY, sorted from least to most cardinality.
+</target_datasource_instructions>
+<target_datasource_content>
+SCHEMA >
+    `total_sales` AggregateFunction(sum, Float64),
+    `sales_count` AggregateFunction(count, UInt64),
+    `column_name_2` AggregateFunction(avg, Float64),
+    `dimension_1` String,
+    `dimension_2` String,
+    ...
+    `date` DateTime
+
+ENGINE "AggregatingMergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(date)"
+ENGINE_SORTING_KEY "date, dimension_1, dimension_2, ..."
+</target_datasource_content>
 """
 
 
@@ -821,6 +843,10 @@ Follow these instructions when creating or updating .pipe files:
 Follow these instructions when creating or updating .yaml files for tests:
 {test_instructions}
 </test_file_instructions>
+<deployment_instruction>
+Follow these instructions when evolving a datasource schema:
+{deployment_instructions}
+</deployment_instruction>
 """.format(
         base_command=base_command,
         datasource_instructions=datasource_instructions,
@@ -831,6 +857,7 @@ Follow these instructions when creating or updating .yaml files for tests:
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
         test_instructions=test_instructions,
+        deployment_instructions=deployment_instructions,
     )
 
 
@@ -879,3 +906,24 @@ test_instructions = """
 
 </test_file_format>
 """
+
+deployment_instructions = """
+- When you make schema changes that are incompatible with the old schema, you must use a forward query in your data source. Forward queries are necessary when introducing breaking changes. Otherwise, your deployment will fail due to a schema mismatch.
+- Forward queries translate the old schema to a new one that you define in the .datasource file. This helps you evolve your schema while continuing to ingest data.
+Follow these steps to evolve your schema using a forward query:
+- Edit the .datasource file to add a forward query.
+- Run tb deploy --check to validate the deployment before creating it.
+- Deploy and promote your changes in Tinybird Cloud using {base_command} --cloud deploy.
+    <forward_query_example>
+SCHEMA >
+    `timestamp` DateTime `json:$.timestamp`,
+    `session_id` UUID `json:$.session_id`,
+    `action` String `json:$.action`,
+    `version` String `json:$.version`,
+    `payload` String `json:$.payload`
+
+FORWARD_QUERY >
+    select timestamp, toUUID(session_id) as session_id, action, version, payload 
+    </forward_query_example>
+</deployment_instruction>
+"""

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev72'
-__revision__ = 'f2d1d8b'
+__version__ = '0.0.1.dev74'
+__revision__ = '68ada75'

tinybird/tb/cli.py

@@ -20,6 +20,7 @@ import tinybird.tb.modules.login
 import tinybird.tb.modules.materialization
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.pipe
+import tinybird.tb.modules.playground
 import tinybird.tb.modules.tag
 import tinybird.tb.modules.test
 import tinybird.tb.modules.token

tinybird/tb/modules/build.py

@@ -7,6 +7,7 @@ import time
 from functools import partial
 from pathlib import Path
 from typing import Callable, Optional
+from urllib.parse import urlencode
 
 import click
 import requests
@@ -41,7 +42,7 @@ def build(ctx: click.Context, watch: bool) -> None:
         )
 
 
-@cli.command("dev", help="Build the project server side and watch for changes")
+@cli.command("dev", help="Build the project server side and watch for changes.")
 @click.pass_context
 def dev(ctx: click.Context) -> None:
     project: Project = ctx.ensure_object(dict)["project"]
@@ -191,6 +192,7 @@ def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
     table_name = diff
     resource_path = Path(filename)
     resource_name = resource_path.stem
+    resource_content = resource_path.read_text()
 
     pipeline = resource_name if filename.endswith(".pipe") else None
 
@@ -201,6 +203,17 @@ def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
 
     res = asyncio.run(tb_client.query(sql, pipeline=pipeline))
     print_table_formatted(res, table_name)
+    if Project.is_endpoint(resource_content):
+        example_params = {
+            "format": "json",
+            "pipe": resource_name,
+            "q": "",
+            "token": tb_client.token,
+        }
+        endpoint_url = asyncio.run(tb_client._req(f"/examples/query.http?{urlencode(example_params)}"))
+        if endpoint_url:
+            endpoint_url = endpoint_url.replace("http://localhost:8001", tb_client.host)
+            click.echo(FeedbackManager.gray(message="\nTest endpoint at ") + FeedbackManager.info(message=endpoint_url))
 
 
 def process(
@@ -240,7 +253,7 @@ def process(
 def run_watch(
     project: Project, tb_client: TinyB, process: Callable[[bool, Optional[str], Optional[str]], None]
 ) -> None:
-    shell = Shell(project=project, tb_client=tb_client)
+    shell = Shell(project=project, tb_client=tb_client, playground=False)
     click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
     watcher_thread = threading.Thread(
         target=watch_project,

tinybird/tb/modules/cli.py

@@ -52,11 +52,11 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
     default=False,
     help="Prints internal representation, can be combined with any command to get more information.",
 )
-@click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
+@click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file.")
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
-@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
-@click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local")
-@click.option("--build", is_flag=True, default=False, help="Run against build mode")
+@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens.")
+@click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
+@click.option("--build", is_flag=True, default=False, help="Run against build mode.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
 @click.version_option(version=VERSION)
 @click.pass_context
@@ -189,7 +189,7 @@ async def dependencies(
 
 @cli.command(
     name="diff",
-    short_help="Diff local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
+    short_help="Diff local datafiles to the corresponding remote files in the workspace. Only diffs VERSION and SCHEMA for .datasource files.",
 )
 @click.argument("filename", type=click.Path(exists=True), nargs=-1, required=False)
 @click.option(
@@ -394,7 +394,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
     if command in commands_without_ctx_client:
         return None
 
-    commands_always_cloud = ["pull"]
+    commands_always_cloud = ["pull", "playground"]
     commands_always_build = ["build", "test", "dev"]
     commands_always_local = ["create"]
     if (

tinybird/tb/modules/common.py

@@ -244,8 +244,7 @@ class CatchAuthExceptions(AliasedGroup):
         formatter.write_text(
             """
   Tinybird collects anonymous usage data and errors to improve the command
-line experience. To opt-out, set TB_CLI_TELEMETRY_OPTOUT environment
-variable to '1' or 'true'."""
+line experience. To opt-out, set TB_CLI_TELEMETRY_OPTOUT to '1' or 'true'."""
         )
         formatter.write_paragraph()
 

tinybird/tb/modules/config.py

@@ -77,7 +77,7 @@ class CLIConfig:
     _global: Optional["CLIConfig"] = None
     _projects: Dict[str, "CLIConfig"] = {}
 
-    def __init__(self, path: Optional[str], parent: Optional["CLIConfig"] = None) -> None:
+    def __init__(self, path: Optional[str] = None, parent: Optional["CLIConfig"] = None) -> None:
         self._path = path
         self._parent = parent
         self._values: Dict[str, ConfigValue] = {}

tinybird/tb/modules/copy.py

@@ -21,7 +21,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 @cli.group()
 @click.pass_context
 def copy(ctx):
-    """Copy pipe commands"""
+    """Copy pipe commands."""
 
 
 @copy.command(name="ls")

tinybird/tb/modules/datafile/build_pipe.py

@@ -9,12 +9,11 @@ import requests
 from croniter import croniter
 
 from tinybird.client import DoesNotExistException, TinyB
-from tinybird.tb.modules.common import getenv_bool, requests_delete, requests_get, wait_job
+from tinybird.tb.modules.common import requests_delete, requests_get, wait_job
+from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.common import ON_DEMAND, CopyModes, CopyParameters, PipeNodeTypes, PipeTypes
-from tinybird.tb.modules.datafile.pipe_checker import PipeCheckerRunner
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.table import format_pretty_table
 
 
 async def new_pipe(
@@ -47,7 +46,8 @@ async def new_pipe(
 ):
     # TODO use tb_client instead of calling the urls directly.
     host = tb_client.host
-    token = tb_client.token
+    config = CLIConfig().get_project_config()
+    token = config.get_user_token()
 
     headers = {"Authorization": f"Bearer {token}"}
 
@@ -55,11 +55,16 @@ async def new_pipe(
     cli_params["cli_version"] = tb_client.version
     cli_params["description"] = p.get("description", "")
     cli_params["ignore_sql_errors"] = "true" if ignore_sql_errors else "false"
+    cli_params["workspace_id"] = config.get("id", None)
 
-    r: requests.Response = await requests_get(f"{host}/v0/pipes/{p['name']}?{urlencode(cli_params)}", headers=headers)
-
-    current_pipe = r.json() if r.status_code == 200 else None
-    pipe_exists = current_pipe is not None
+    r: requests.Response = await requests_get(f"{host}/v0/playgrounds?{urlencode(cli_params)}", headers=headers)
+    current_pipe = None
+    pipe_exists = False
+    playgrounds_response = r.json() if r.status_code == 200 else None
+    if playgrounds_response:
+        playgrounds = playgrounds_response["playgrounds"]
+        current_pipe = next((play for play in playgrounds if play["name"] == p["name"] + "__tb__playground"), None)
+        pipe_exists = current_pipe is not None
 
     is_materialized = any([node.get("params", {}).get("type", None) == "materialized" for node in p["nodes"]])
     copy_node = next((node for node in p["nodes"] if node.get("params", {}).get("type", None) == "copy"), None)
@@ -75,27 +80,7 @@ async def new_pipe(
             # TODO: this should create a different node and rename it to the final one on success
             if check and not populate:
                 if not is_materialized and not copy_node and not sink_node and not stream_node:
-                    await check_pipe(
-                        p,
-                        host,
-                        token,
-                        populate,
-                        tb_client,
-                        only_response_times=only_response_times,
-                        limit=tests_to_run,
-                        relative_change=tests_relative_change,
-                        sample_by_params=tests_to_sample_by_params,
-                        matches=tests_filter_by,
-                        failfast=tests_failfast,
-                        validate_processed_bytes=tests_validate_processed_bytes,
-                        ignore_order=tests_ignore_order,
-                        token_for_requests_to_check=(
-                            await get_token_from_main_branch(tb_client)
-                            if not tests_check_requests_from_branch
-                            else None
-                        ),
-                        current_pipe=current_pipe,
-                    )
+                    pass
                 else:
                     if is_materialized:
                         await check_materialized(
@@ -167,9 +152,27 @@ async def new_pipe(
     post_headers.update(headers)
 
     try:
-        data = await tb_client._req(
-            f"/v0/pipes?{urlencode(params)}", method="POST", headers=post_headers, data=json.dumps(body)
-        )
+        user_client = deepcopy(tb_client)
+        config = CLIConfig().get_project_config()
+        user_client.token = config.get_user_token()
+        params["workspace_id"] = config.get("id", None)
+        body["name"] = p["name"] + "__tb__playground"
+
+        if pipe_exists and current_pipe:
+            data = await user_client._req(
+                f"/v0/playgrounds/{current_pipe['id']}?{urlencode(params)}",
+                method="PUT",
+                headers=post_headers,
+                data=json.dumps(body),
+            )
+
+        else:
+            data = await user_client._req(
+                f"/v0/playgrounds?{urlencode(params)}",
+                method="POST",
+                headers=post_headers,
+                data=json.dumps(body),
+            )
     except Exception as e:
         raise click.ClickException(FeedbackManager.error_pushing_pipe(pipe=p["name"], error=str(e)))
 
@@ -296,160 +299,6 @@ async def get_token_from_main_branch(branch_tb_client: TinyB) -> Optional[str]:
     return token_from_main_branch
 
 
-async def check_pipe(
-    pipe,
-    host: str,
-    token: str,
-    populate: bool,
-    cl: TinyB,
-    limit: int = 0,
-    relative_change: float = 0.01,
-    sample_by_params: int = 0,
-    only_response_times=False,
-    matches: Optional[List[str]] = None,
-    failfast: bool = False,
-    validate_processed_bytes: bool = False,
-    ignore_order: bool = False,
-    token_for_requests_to_check: Optional[str] = None,
-    current_pipe: Optional[Dict[str, Any]] = None,
-):
-    checker_pipe = deepcopy(pipe)
-    checker_pipe["name"] = f"{checker_pipe['name']}__checker"
-
-    if current_pipe:
-        pipe_type = current_pipe["type"]
-        if pipe_type == PipeTypes.COPY:
-            await cl.pipe_remove_copy(current_pipe["id"], current_pipe["copy_node"])
-        if pipe_type == PipeTypes.DATA_SINK:
-            await cl.pipe_remove_sink(current_pipe["id"], current_pipe["sink_node"])
-        if pipe_type == PipeTypes.STREAM:
-            await cl.pipe_remove_stream(current_pipe["id"], current_pipe["stream_node"])
-
-    # In case of doing --force for a materialized view, checker is being created as standard pipe
-    for node in checker_pipe["nodes"]:
-        node["params"]["type"] = PipeNodeTypes.STANDARD
-
-    if populate:
-        raise click.ClickException(FeedbackManager.error_check_pipes_populate())
-
-    runner = PipeCheckerRunner(pipe["name"], host)
-    headers = (
-        {"Authorization": f"Bearer {token_for_requests_to_check}"}
-        if token_for_requests_to_check
-        else {"Authorization": f"Bearer {token}"}
-    )
-
-    sql_for_coverage, sql_latest_requests = runner.get_sqls_for_requests_to_check(
-        matches or [], sample_by_params, limit
-    )
-
-    params = {"q": sql_for_coverage if limit == 0 and sample_by_params > 0 else sql_latest_requests}
-    r: requests.Response = await requests_get(
-        f"{host}/v0/sql?{urlencode(params)}", headers=headers, verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False)
-    )
-
-    # If we get a timeout, fallback to just the last requests
-
-    if not r or r.status_code == 408:
-        params = {"q": sql_latest_requests}
-        r = await requests_get(
-            f"{host}/v0/sql?{urlencode(params)}",
-            headers=headers,
-            verify=not getenv_bool("TB_DISABLE_SSL_CHECKS", False),
-        )
-
-    if not r or r.status_code != 200:
-        raise click.ClickException(FeedbackManager.error_check_pipes_api(pipe=pipe["name"]))
-
-    pipe_requests_to_check: List[Dict[str, Any]] = []
-    for row in r.json().get("data", []):
-        for i in range(len(row["endpoint_url"])):
-            pipe_requests_to_check += [
-                {
-                    "endpoint_url": f"{host}{row['endpoint_url'][i]}",
-                    "pipe_request_params": row["pipe_request_params"][i],
-                    "http_method": row["http_method"],
-                }
-            ]
-
-    if not pipe_requests_to_check:
-        return
-
-    await new_pipe(checker_pipe, cl, force=True, check=False, populate=populate)
-
-    runner_response = runner.run_pipe_checker(
-        pipe_requests_to_check,
-        checker_pipe["name"],
-        token,
-        only_response_times,
-        ignore_order,
-        validate_processed_bytes,
-        relative_change,
-        failfast,
-    )
-
-    try:
-        if runner_response.metrics_summary and runner_response.metrics_timing:
-            column_names_tests = ["Test Run", "Test Passed", "Test Failed", "% Test Passed", "% Test Failed"]
-            click.echo("\n==== Test Metrics ====\n")
-            click.echo(
-                format_pretty_table(
-                    [
-                        [
-                            runner_response.metrics_summary["run"],
-                            runner_response.metrics_summary["passed"],
-                            runner_response.metrics_summary["failed"],
-                            runner_response.metrics_summary["percentage_passed"],
-                            runner_response.metrics_summary["percentage_failed"],
-                        ]
-                    ],
-                    column_names=column_names_tests,
-                )
-            )
-
-            column_names_timing = ["Timing Metric (s)", "Current", "New"]
-            click.echo("\n==== Response Time Metrics ====\n")
-            click.echo(
-                format_pretty_table(
-                    [
-                        [metric, runner_response.metrics_timing[metric][0], runner_response.metrics_timing[metric][1]]
-                        for metric in [
-                            "min response time",
-                            "max response time",
-                            "mean response time",
-                            "median response time",
-                            "p90 response time",
-                            "min read bytes",
-                            "max read bytes",
-                            "mean read bytes",
-                            "median read bytes",
-                            "p90 read bytes",
-                        ]
-                    ],
-                    column_names=column_names_timing,
-                )
-            )
-    except Exception:
-        pass
-
-    if not runner_response.was_successfull:
-        for failure in runner_response.failed:
-            try:
-                click.echo("==== Test FAILED ====\n")
-                click.echo(failure["name"])
-                click.echo(FeedbackManager.error_check_pipe(error=failure["error"]))
-                click.echo("=====================\n\n\n")
-            except Exception:
-                pass
-        raise RuntimeError("Invalid results, you can bypass checks by running push with the --no-check flag")
-
-    # Only delete if no errors, so we can check results after failure
-    headers = {"Authorization": f"Bearer {token}"}
-    r = await requests_delete(f"{host}/v0/pipes/{checker_pipe['name']}", headers=headers)
-    if r.status_code != 204:
-        click.echo(FeedbackManager.warning_check_pipe(content=r.content))
-
-
 async def check_materialized(pipe, host, token, cl, override_datasource=False, current_pipe=None):
     checker_pipe = deepcopy(pipe)
     checker_pipe["name"] = f"{checker_pipe['name']}__checker"

tinybird/tb/modules/datafile/common.py

@@ -1151,6 +1151,21 @@ def parse(
 
         return raise_deprecation_error
 
+    def not_supported_yet(extra_message: str = "") -> Callable[..., Any]:
+        def inner(func: Callable[..., Any]) -> Callable[..., Any]:
+            @functools.wraps(func)
+            def raise_not_supported_yet_error(*args: Any, **kwargs: Any) -> Any:
+                extra_message_bit = f". {extra_message}" if extra_message else ""
+                raise DatafileSyntaxError(
+                    f"{kwargs['cmd']} is not supported yet{extra_message_bit}",
+                    lineno=kwargs["lineno"],
+                    pos=1,
+                )
+
+            return raise_not_supported_yet_error
+
+        return inner
+
     def assign(attr):
         @multiline_not_supported
         def _fn(x, **kwargs):
@@ -1255,6 +1270,7 @@ def parse(
 
         return _f
 
+    @not_supported_yet(extra_message="You can manage tokens with `tb token` in the meantime.")
     @multiline_not_supported
     def add_token(*args: str, **kwargs: Any) -> None:  # token_name, permissions):
         # lineno = kwargs["lineno"]
@@ -1268,6 +1284,7 @@ def parse(
         # APPEND for datasources
         doc.tokens.append({"token_name": _unquote(args[0]), "permissions": args[1]})
 
+    @not_supported_yet()
     def include(*args: str, **kwargs: Any) -> None:
         f = _unquote(args[0])
         f = eval_var(f)