tinybird 0.0.1.dev15__tar.gz → 0.0.1.dev16__tar.gz

{tinybird-0.0.1.dev15 → tinybird-0.0.1.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev15
+Version: 0.0.1.dev16
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev15 → tinybird-0.0.1.dev16}/tinybird/client.py

@@ -188,7 +188,9 @@ class TinyB:
         if response.status_code == 599:
             raise TimeoutException("timeout")
         if "Content-Type" in response.headers and (
-            response.headers["Content-Type"] == "text/plain" or "text/csv" in response.headers["Content-Type"]
+            response.headers["Content-Type"] == "text/plain"
+            or "text/csv" in response.headers["Content-Type"]
+            or "ndjson" in response.headers["Content-Type"]
         ):
             return response.content.decode("utf-8")
         if response.status_code >= 400 and response.status_code not in [400, 403, 404, 409, 429]:

{tinybird-0.0.1.dev15 → tinybird-0.0.1.dev16}/tinybird/feedback_manager.py

@@ -1,5 +1,6 @@
 from collections import namedtuple
 from typing import Any, Callable
+import os
 
 FeedbackMessage = namedtuple("FeedbackMessage", "message")
 
@@ -26,6 +27,16 @@ def print_message(message: str, color: str = bcolors.ENDC) -> Callable[..., str]
 def error_message(message: str) -> Callable[..., str]:
     return print_message(f"\n** {message}", bcolors.FAIL)
 
+def error_exception(message: Exception) -> Callable[..., str]:
+    def formatter(**kwargs: Any) -> str:
+        color = bcolors.FAIL
+        if isinstance(kwargs.get('error', ''), Exception) and os.environ.get('TB_DEBUG', '') != '':
+            import traceback
+            stack_trace = f"Traceback:\n{''.join(traceback.format_tb(kwargs['error'].__traceback__))}"
+            return f"{color}{message.format(**kwargs)}{stack_trace}{bcolors.ENDC}"
+        return f"{color}{message.format(**kwargs)}{bcolors.ENDC}"
+    return formatter
+
 
 def simple_error_message(message: str) -> Callable[..., str]:
     return print_message(f"{message}", bcolors.FAIL)
@@ -56,7 +67,7 @@ def gray_message(message: str) -> Callable[..., str]:
 
 
 class FeedbackManager:
-    error_exception = error_message("{error}")
+    error_exception = error_exception("{error}")
     simple_error_exception = simple_error_message("{error}")
     error_exception_trace = error_message("{error}\n** Trace:\n{trace}")
     error_notoken = error_message(

{tinybird-0.0.1.dev15/tinybird/tb/modules → tinybird-0.0.1.dev16/tinybird}/prompts.py

@@ -76,68 +76,7 @@ SQL >
 </instructions>
 """
 
-create_test_calls_prompt = """
-You are a Tinybird expert. You will be given a pipe endpoint containing different nodes with SQL and Tinybird templating syntax. You will generate URLs to test it with different parameters combinations.
-
-<test>
-    <test_1>:
-        name: <test_name_1>
-        description: <description_1>
-        parameters: <url_encoded_parameters_1>
-    <test_2>:
-        name: <test_name_2>
-        description: <description_2>
-        parameters: <url_encoded_parameters_2>
-</test>
-<instructions>
-    - The test name must be unique.
-    - The test command must be a valid Tinybird command that can be run in the terminal.
-    - The test command can have as many parameters as are needed to test the pipe.
-    - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
-    - If there are no parameters in the , you can omit parametrs and generate a single test command.
-    - Extra context: {context}
-</instructions>
-"""
-
-sample_data_prompt = """Your role is to generate sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that fit that schema.
-
-The schema will be provided to you at the end of this prompt in the following format:
-
-```
-SCHEMA >
-    `span_id` String `json:$.span_id`,
-    `operation_id` String `json:$.operation_name`,
-    `duration` Nullable(Float32) `json:$.duration`,
-```
-
-In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
-
-You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will only contain the {row_count} rows of data, without any other text. Do not format the data in any way, and do not place it inside any markers for a code block of any kind. Most importantly, do not use backticks on the first or last line or anywhere else in the response. This is extremely important, ensure that your response doesn't include any backticks.
-
-Remember that for a JSON to be valid, there can't be a trailing comma after the last value in an object.
-
-Make sure to include all columns in the schema, and use valid JSON. You may leave some columns empty only if the column type allows it (e.g. Nullable(String)). Use realistic values for the data, and make sure to include a variety of values in each column.
-
-For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
-
-sample_data_with_errors_prompt = """Your role is to generate incorrect sample data for a given data source. A data source is a table in Tinybird. You will be given a schema for the data source, and you will need to generate a set of rows that mostly fit that schema, but with some errors.
-
-The schema will be provided to you at the end of this prompt in the following format:
-
-```
-SCHEMA >
-    `span_id` String `json:$.span_id`,
-    `operation_id` String `json:$.operation_name`,
-    `duration` Nullable(Float32) `json:$.duration`,
-```
-
-In the example above, `span_id` are column names, String and Float32 are column types, and `json:$.duration` is a JSON path expression that points to the duration value in the JSON data. Use this information to generate valid data.
-
-You will generate {row_count} rows of data in JSON format, newlines separated. Your reponse will be a single string containing all {row_count} rows, without any other text. Make sure to include all columns in the schema, and use valid JSON. Use realistic values for the data, and make sure to include a variety of values in each column. In every set of {row_count} rows, there should be at least one row that has an error. The error could be in any column, and it could be a type error (e.g. a string where a number is expected), or it could be an error in the structure of the JSON (e.g. a value in the wrong place in the document).
-
-For columns that represent timestamps, use values that are close to the current date and time. The current date and time is {current_datetime}"""
-
-sample_data_sql_prompt = """
+generate_sql_mock_data_prompt = """
 Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
 
 Response format MUST be just a valid clickhouse sql query.
@@ -164,13 +103,13 @@ SELECT
     rand() % 50 AS pvp_kills,           -- Random PvP kills between 0 and 49
     rand() % 200 AS quest_completions,  -- Random quest completions between 0 and 199
     now() - rand() % 86400 AS timestamp -- Random timestamp within the last day
-FROM numbers({row_count})
+FROM numbers({rows})
 
 # Instructions:
 
 - The query MUST return a random sample of data that matches the schema.
 - The query MUST return a valid clickhouse sql query.
-- The query MUST return a sample of EXACTLY {row_count} rows.
+- The query MUST return a sample of EXACTLY {rows} rows.
 - The query MUST be valid for clickhouse and Tinybird.
 - Return JUST the sql query, without any other text or symbols. 
 - Do NOT include ```clickhouse or ```sql or any other wrapping text.
@@ -182,3 +121,26 @@ FROM numbers({row_count})
 {context}
 
 """
+
+create_test_calls_prompt = """
+You are a Tinybird expert. You will be given a pipe endpoint containing different nodes with SQL and Tinybird templating syntax. You will generate URLs to test it with different parameters combinations.
+
+<test>
+    <test_1>:
+        name: <test_name_1>
+        description: <description_1>
+        parameters: <url_encoded_parameters_1>
+    <test_2>:
+        name: <test_name_2>
+        description: <description_2>
+        parameters: <url_encoded_parameters_2>
+</test>
+<instructions>
+    - The test name must be unique.
+    - The test command must be a valid Tinybird command that can be run in the terminal.
+    - The test command can have as many parameters as are needed to test the pipe.
+    - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
+    - If there are no parameters in the , you can omit parametrs and generate a single test command.
+    - Extra context: {context}
+</instructions>
+"""

{tinybird-0.0.1.dev15 → tinybird-0.0.1.dev16}/tinybird/tb/cli.py

@@ -14,10 +14,12 @@ import tinybird.tb.modules.create
 import tinybird.tb.modules.datasource
 import tinybird.tb.modules.fmt
 import tinybird.tb.modules.job
+import tinybird.tb.modules.local
 import tinybird.tb.modules.login
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag
+import tinybird.tb.modules.test
 import tinybird.tb.modules.token
 import tinybird.tb.modules.workspace
 import tinybird.tb.modules.workspace_members

{tinybird-0.0.1.dev15 → tinybird-0.0.1.dev16}/tinybird/tb/modules/build.py

@@ -1,26 +1,20 @@
 import asyncio
-import cmd
 import os
-import random
-import subprocess
-import sys
 import threading
 import time
 from pathlib import Path
 from typing import Any, Awaitable, Callable, List, Union
 
 import click
-import humanfriendly
 from click import Context
 from watchdog.events import FileSystemEventHandler
 from watchdog.observers import Observer
 
 import tinybird.context as context
-from tinybird.syncasync import async_to_sync
-from tinybird.client import TinyB, AuthNoTokenException
-from tinybird.tb.modules.common import CLIException
+from tinybird.client import TinyB
 from tinybird.config import FeatureFlags
-from tinybird.feedback_manager import FeedbackManager, bcolors
+from tinybird.feedback_manager import FeedbackManager
+from tinybird.tb.modules.build_shell import BuildShell, print_table_formatted
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import coro, push_data
 from tinybird.tb.modules.datafile.build import folder_build
@@ -29,48 +23,7 @@ from tinybird.tb.modules.datafile.exceptions import ParseException
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
-from tinybird.tb.modules.local import get_tinybird_local_client
-from tinybird.tb.modules.table import format_table
-
-
-class BuildShell(cmd.Cmd):
-    prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
-
-    def __init__(self, folder: str, client: TinyB):
-        super().__init__()
-        self.folder = folder
-        self.client = client
-    def do_exit(self, arg):
-        sys.exit(0)
-
-    def do_quit(self, arg):
-        sys.exit(0)
-
-    def default(self, argline):
-        click.echo("")
-        if argline.startswith("tb build"):
-            click.echo(FeedbackManager.error(message="Build command is already running"))
-        else:
-            arg_stripped = argline.strip().lower()
-            if not arg_stripped:
-                return
-            if arg_stripped.startswith("tb"):
-                extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
-                subprocess.run(arg_stripped + extra_args, shell=True, text=True)
-            elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
-                try:
-                    run_sql(self.client, argline)
-                except Exception as e:
-                    click.echo(FeedbackManager.error(message=str(e)))
-                
-            elif arg_stripped.startswith("mock "):
-                subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
-            else:
-                click.echo(FeedbackManager.error(message="Invalid command"))
-
-    def reprint_prompt(self):
-        self.stdout.write(self.prompt)
-        self.stdout.flush()
+from tinybird.tb.modules.local_common import get_tinybird_local_client
 
 
 class FileChangeHandler(FileSystemEventHandler):
@@ -131,44 +84,6 @@ def watch_files(
     observer.join()
 
 
-
-def run_sql(client: TinyB, query, rows_limit=20):
-    try:
-        q = query.strip()
-        if q.startswith("insert"):
-            click.echo(FeedbackManager.info_append_data())
-            raise CLIException(FeedbackManager.error_invalid_query())
-        if q.startswith("delete"):
-            raise CLIException(FeedbackManager.error_invalid_query())
-
-        # fuck my life
-        def run_query_in_thread():
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            try:
-                return loop.run_until_complete(client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON"))
-            finally:
-                loop.close()
-
-        # Run the query in a separate thread
-        import concurrent.futures
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            res = executor.submit(run_query_in_thread).result()
-
-    except AuthNoTokenException:
-        raise
-    except Exception as e:
-        raise CLIException(FeedbackManager.error_exception(error=str(e)))
-
-    if isinstance(res, dict) and "error" in res:
-        raise CLIException(FeedbackManager.error_exception(error=res["error"]))
-
-    if isinstance(res, dict) and "data" in res and res["data"]:
-        print_table_formatted(res, 'QUERY')
-    else:
-        click.echo(FeedbackManager.info_no_rows())
-
-
 @cli.command()
 @click.option(
     "--folder",
@@ -286,46 +201,6 @@ async def build_and_print_resource(tb_client: TinyB, filename: str):
     res = await tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline)
     print_table_formatted(res, name)
 
-def print_table_formatted(res: dict, name: str):
-    rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
-    rebuild_index = random.randint(0, len(rebuild_colors) - 1)
-    rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
-    data = []
-    limit = 5
-    for d in res["data"][:5]:
-        data.append(d.values())
-    meta = res["meta"]
-    row_count = res.get("rows", 0)
-    stats = res.get("statistics", {})
-    elapsed = stats.get("elapsed", 0)
-    cols = len(meta)
-    try:
-
-        def print_message(message: str, color=bcolors.CGREY):
-            return f"{color}{message}{bcolors.ENDC}"
-
-        table = format_table(data, meta)
-        colored_char = print_message("│", rebuild_color)
-        table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
-        click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
-        click.echo(colored_char)
-        click.echo(table_with_marker)
-        click.echo(colored_char)
-        rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
-        bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
-        elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
-        stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
-        rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
-        click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
-        click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
-    except ValueError as exc:
-        if str(exc) == "max() arg is an empty sequence":
-            click.echo("------------")
-            click.echo("Empty")
-            click.echo("------------")
-        else:
-            raise exc
-
 
 async def append_datasource(
     ctx: click.Context,

tinybird-0.0.1.dev16/tinybird/tb/modules/build_shell.py

@@ -0,0 +1,133 @@
+import asyncio
+import cmd
+import random
+import subprocess
+import sys
+
+import click
+import humanfriendly
+
+from tinybird.client import TinyB
+from tinybird.feedback_manager import FeedbackManager, bcolors
+from tinybird.tb.modules.exceptions import CLIException
+from tinybird.tb.modules.table import format_table
+
+
+class BuildShell(cmd.Cmd):
+    prompt = "\n\001\033[1;32m\002TB > \001\033[0m\002"
+
+    def __init__(self, folder: str, client: TinyB):
+        super().__init__()
+        self.folder = folder
+        self.client = client
+
+    def do_exit(self, arg):
+        sys.exit(0)
+
+    def do_quit(self, arg):
+        sys.exit(0)
+
+    def default(self, argline):
+        click.echo("")
+        if argline.startswith("tb build"):
+            click.echo(FeedbackManager.error(message="Build command is already running"))
+        else:
+            arg_stripped = argline.strip().lower()
+            if not arg_stripped:
+                return
+            if arg_stripped.startswith("tb"):
+                arg_stripped = arg_stripped.replace("tb", "tb --local")
+                extra_args = f" --folder {self.folder}" if arg_stripped.startswith("tb mock") else ""
+                subprocess.run(arg_stripped + extra_args, shell=True, text=True)
+            elif arg_stripped.startswith("with") or arg_stripped.startswith("select"):
+                try:
+                    self.run_sql(self.client, argline)
+                except Exception as e:
+                    click.echo(FeedbackManager.error(message=str(e)))
+
+            elif arg_stripped.startswith("mock "):
+                subprocess.run(f"tb {arg_stripped} --folder {self.folder}", shell=True, text=True)
+            else:
+                click.echo(FeedbackManager.error(message="Invalid command"))
+
+    def reprint_prompt(self):
+        self.stdout.write(self.prompt)
+        self.stdout.flush()
+
+    def run_sql(self, query, rows_limit=20):
+        try:
+            q = query.strip()
+            if q.startswith("insert"):
+                click.echo(FeedbackManager.info_append_data())
+                raise CLIException(FeedbackManager.error_invalid_query())
+            if q.startswith("delete"):
+                raise CLIException(FeedbackManager.error_invalid_query())
+
+            # fuck my life
+            def run_query_in_thread():
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+                try:
+                    return loop.run_until_complete(
+                        self.client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON")
+                    )
+                finally:
+                    loop.close()
+
+            # Run the query in a separate thread
+            import concurrent.futures
+
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                res = executor.submit(run_query_in_thread).result()
+
+        except Exception as e:
+            raise CLIException(FeedbackManager.error_exception(error=str(e)))
+
+        if isinstance(res, dict) and "error" in res:
+            raise CLIException(FeedbackManager.error_exception(error=res["error"]))
+
+        if isinstance(res, dict) and "data" in res and res["data"]:
+            print_table_formatted(res, "QUERY")
+        else:
+            click.echo(FeedbackManager.info_no_rows())
+
+
+def print_table_formatted(res: dict, name: str):
+    rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
+    rebuild_index = random.randint(0, len(rebuild_colors) - 1)
+    rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
+    data = []
+    limit = 5
+    for d in res["data"][:5]:
+        data.append(d.values())
+    meta = res["meta"]
+    row_count = res.get("rows", 0)
+    stats = res.get("statistics", {})
+    elapsed = stats.get("elapsed", 0)
+    cols = len(meta)
+    try:
+
+        def print_message(message: str, color=bcolors.CGREY):
+            return f"{color}{message}{bcolors.ENDC}"
+
+        table = format_table(data, meta)
+        colored_char = print_message("│", rebuild_color)
+        table_with_marker = "\n".join(f"{colored_char} {line}" for line in table.split("\n"))
+        click.echo(f"\n{colored_char} {print_message('⚡', rebuild_color)} Running {name}")
+        click.echo(colored_char)
+        click.echo(table_with_marker)
+        click.echo(colored_char)
+        rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
+        bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
+        elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
+        stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
+        rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
+        click.echo(f"{colored_char} {print_message(stats_message, bcolors.OKGREEN)}")
+        click.echo(f"{colored_char} {print_message(rows_message, bcolors.CGREY)}")
+    except ValueError as exc:
+        if str(exc) == "max() arg is an empty sequence":
+            click.echo("------------")
+            click.echo("Empty")
+            click.echo("------------")
+        else:
+            raise exc

{tinybird-0.0.1.dev15 → tinybird-0.0.1.dev16}/tinybird/tb/modules/cli.py

@@ -63,6 +63,7 @@ from tinybird.tb.modules.datafile.exceptions import (
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull
+from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.telemetry import add_telemetry_event
 
 __old_click_echo = click.echo
@@ -108,6 +109,7 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
     "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
 )
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
+@click.option("--local", is_flag=True, default=False, help="Run in local mode")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro
@@ -131,6 +133,7 @@ async def cli(
     sf_stage,
     with_headers: bool,
     show_tokens: bool,
+    local: bool,
 ) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
@@ -228,7 +231,11 @@ async def cli(
 
     logging.debug("debug enabled")
 
-    ctx.ensure_object(dict)["client"] = _get_tb_client(config.get("token", None), config["host"], semver)
+    ctx.ensure_object(dict)["client"] = (
+        await get_tinybird_local_client()
+        if local
+        else _get_tb_client(config.get("token", None), config["host"], semver)
+    )
 
     for connector in SUPPORTED_CONNECTORS:
         load_connector_config(ctx, connector, debug, check_uninstalled=True)