tinybird 0.0.1.dev148__py3-none-any.whl → 0.0.1.dev152__py3-none-any.whl

tinybird/prompts.py

@@ -401,7 +401,7 @@ Follow the instructions and generate the following response with no additional t
 def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
     feedback_history = ""
     if feedback and history:
-        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
 it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
 Use the following feedback and history to regenerate the response:
 Feedback to improve the response:
@@ -420,7 +420,9 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 
 {feedback_history}
 
@@ -443,14 +445,16 @@ Use the following format to generate the response and do not wrap it in any othe
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
         feedback_history=feedback_history,
     )
 
 
 def mock_prompt(rows: int, feedback: str = "") -> str:
     if feedback:
-        feedback = f"""In case the <feedback> tag is present and not empty, 
+        feedback = f"""In case the <feedback> tag is present and not empty,
 it means there was a previous attempt to generate the resources and the system provided feedback about the previous response.
 Use the following feedback to regenerate the response:
 <feedback>{feedback}</feedback>
@@ -515,7 +519,7 @@ SCHEMA >
     `customer_id` UInt64 `json:$.customer_id`,
     `order_date` DateTime `json:$.order_date`,
     `total_amount` Float64 `json:$.total_amount`,
-    `items` Array(String) `json:$.items[:]` // This is an array field 
+    `items` Array(String) `json:$.items[:]` // This is an array field
 
 ### Desired final output of the query:
 {{
@@ -680,19 +684,34 @@ connection_instructions = """
     - Content cannot be empty.
     - The connection names must be unique.
     - No indentation is allowed for property names
-    - We only support kafka connections for now
+    - We support kafka, gcs and s3 connections for now
 </connection_file_instructions>
 """
 
-connection_example = """
-<connection_content>
+kafka_connection_example = """
+<kafka_connection_content>
 TYPE kafka
 KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }}
 KAFKA_SECURITY_PROTOCOL SASL_SSL
 KAFKA_SASL_MECHANISM PLAIN
 KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }}
 KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }}
-</connection_content>
+</kafka_connection_content>
+"""
+
+gcs_connection_example = """
+<gcs_connection_content>
+TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON", "") }}
+</gcs_connection_content>
+"""
+
+s3_connection_example = """
+<s3_connection_content>
+TYPE s3
+S3_REGION {{ tb_secret("PRODUCTION_S3_REGION", "") }}
+S3_ARN {{ tb_secret("PRODUCTION_S3_ARN", "") }}
+</s3_connection_content>
 """
 
 datasource_instructions = """
@@ -703,6 +722,7 @@ datasource_instructions = """
     - Use MergeTree engine by default.
     - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
+    - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
 </datasource_file_instructions>
 """
 
@@ -775,7 +795,7 @@ sql_instructions = """
     {{%if not defined(end_date)%}}
     AND now()
     {{%else%}}
-    AND {{{{DateTime(end_date)}}}} 
+    AND {{{{DateTime(end_date)}}}}
     {{%end%}}
     </valid_condition_without_now>
     - Parameters must not be quoted.
@@ -862,7 +882,9 @@ Follow these instructions when creating or updating .pipe files:
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 </pipe_file_instructions>
 <test_file_instructions>
 Follow these instructions when creating or updating .yaml files for tests:
@@ -884,7 +906,9 @@ Follow these instructions when evolving a datasource schema:
         test_instructions=test_instructions,
         deployment_instructions=deployment_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
     )
 
 
@@ -950,7 +974,7 @@ SCHEMA >
     `payload` String `json:$.payload`
 
 FORWARD_QUERY >
-    select timestamp, toUUID(session_id) as session_id, action, version, payload 
+    select timestamp, toUUID(session_id) as session_id, action, version, payload
     </forward_query_example>
 </deployment_instruction>
 """
@@ -968,7 +992,7 @@ Current README.md file:
 - If it is not present in the current readme, generate a new ## Tinybird section with the following content:
     - ### Overview section:
         - Explaining the purpose of the project.
-    - ### Data sources section: 
+    - ### Data sources section:
         - Explaining the purpose of each datasource.
         - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
         curl -X POST "{host}/v0/events?name=events" \

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev148'
-__revision__ = '7b2789f'
+__version__ = '0.0.1.dev152'
+__revision__ = '100f32b'

tinybird/tb/modules/cli.py

@@ -323,16 +323,23 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, s
         return None
 
     commands_always_cloud = ["pull", "infra"]
-    commands_always_local = ["build", "test", "dev"]
-    if (cloud or command in commands_always_cloud) and command not in commands_always_local:
+    commands_always_local = ["build", "dev"]
+    command_always_test = ["test"]
+
+    if (
+        (cloud or command in commands_always_cloud)
+        and command not in commands_always_local
+        and command not in command_always_test
+    ):
         click.echo(
             FeedbackManager.gray(message=f"Running against Tinybird Cloud: Workspace {config.get('name', 'default')}")
         )
         return _get_tb_client(config.get("token", None), config["host"], staging=staging)
     local = command in commands_always_local
+    test = command in command_always_test
     if not local and command not in commands_always_local and command:
         click.echo(FeedbackManager.gray(message="Running against Tinybird Local"))
-    return await get_tinybird_local_client(config, staging=staging)
+    return await get_tinybird_local_client(config, test=test, staging=staging)
 
 
 def get_target_env(cloud: bool) -> str:

tinybird/tb/modules/create.py

@@ -1,3 +1,4 @@
+import glob
 import os
 import re
 from pathlib import Path
@@ -5,7 +6,7 @@ from typing import Any, Dict, Optional, Tuple
 
 import click
 
-from tinybird.prompts import create_prompt, mock_prompt, readme_prompt, rules_prompt
+from tinybird.prompts import create_prompt, readme_prompt, rules_prompt
 from tinybird.tb.client import TinyB
 from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
@@ -17,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
 from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.mock import create_mock_data
 from tinybird.tb.modules.project import Project
 
 
@@ -83,7 +85,7 @@ async def create(
 
         if data or (prompt and user_token):
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result, created_something = await create_resources(tb_client, user_token, data, prompt, folder, ctx_config)
+            result, created_something = await create_resources(tb_client, user_token, data, prompt, project, ctx_config)
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
             if prompt:
                 readme_path = Path(root_folder) / "README.md"
@@ -132,22 +134,26 @@ async def create(
                 persist_fixture(ds_name, data_content, folder, format=data_format)
                 created_something = True
             elif prompt and user_token:
-                datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
-                for datasource_file in datasource_files:
-                    datasource_path = Path(folder) / "datasources" / datasource_file
-                    llm = LLM(user_token=user_token, host=tb_client.host)
+                for datasource_file in project.get_datasource_files():
+                    datasource_path = Path(datasource_file)
                     datasource_name = datasource_path.stem
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
                     if has_json_path:
-                        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
-                        response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
-                        sql = extract_xml(response, "sql")
-                        sql = sql.split("FORMAT")[0]
-                        query_result = await tb_client.query(f"{sql} FORMAT JSON")
-                        data = query_result.get("data", [])
-                        if data:
-                            persist_fixture(datasource_name, data, folder, format="ndjson")
+                        mock_data = await create_mock_data(
+                            datasource_name,
+                            datasource_content,
+                            rows,
+                            prompt,
+                            config,
+                            ctx_config,
+                            user_token,
+                            tb_client,
+                            format_="ndjson",
+                            folder=project.folder,
+                        )
+                        if mock_data:
+                            persist_fixture(datasource_name, mock_data, folder, format="ndjson")
                             click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
                             created_something = True
 
@@ -204,12 +210,12 @@ async def create_resources(
     user_token: Optional[str],
     data: Optional[str],
     prompt: Optional[str],
-    folder: str,
+    project: Project,
     config: Dict[str, Any],
 ) -> Tuple[str, bool]:
     result = ""
     created_any_resource = False
-    folder_path = Path(folder)
+    folder_path = project.path
     if data:
         local_client = await get_tinybird_local_client(config)
         path = folder_path / data
@@ -224,7 +230,7 @@ SQL >
     SELECT * from {name}
 TYPE ENDPOINT
             """,
-            folder,
+            project.folder,
         )
         result = (
             f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
@@ -232,14 +238,9 @@ TYPE ENDPOINT
         created_any_resource = True
 
     elif prompt and user_token:
-        datasource_paths = [
-            Path(folder) / "datasources" / f
-            for f in os.listdir(Path(folder) / "datasources")
-            if f.endswith(".datasource")
-        ]
-        pipes_paths = [
-            Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
-        ]
+        datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
+        pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
+        connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
         resources_xml = "\n".join(
             [
                 f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -254,6 +255,14 @@ TYPE ENDPOINT
                     )
                     for pipe in pipes_paths
                 ]
+                + [
+                    (
+                        "connection",
+                        conn.stem,
+                        conn.read_text(),
+                    )
+                    for conn in connections_paths
+                ]
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)
@@ -287,19 +296,19 @@ TYPE ENDPOINT
                 data=None,
                 _format="ndjson",
                 force=True,
-                folder=folder,
+                folder=project.folder,
             )
             created_any_resource = True
 
         for pipe in pipes:
             content = pipe["content"].replace("```", "")
-            generate_pipe_file(pipe["name"], content, folder)
+            generate_pipe_file(pipe["name"], content, project.folder)
             created_any_resource = True
 
         for conn in connections:
             content = conn["content"].replace("```", "")
             filename = f"{conn['name']}.connection"
-            generate_connection_file(conn["name"], content, folder)
+            generate_connection_file(conn["name"], content, project.folder)
             created_any_resource = True
 
     return result, created_any_resource
@@ -335,21 +344,25 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     def is_endpoint(content: str) -> bool:
         return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
 
-    if is_copy(content):
-        pathname = "copies"
-    elif is_materialization(content):
-        pathname = "materializations"
-    elif is_sink(content):
-        pathname = "sinks"
-    elif is_endpoint(content):
-        pathname = "endpoints"
+    already_exists = glob.glob(f"{folder}/**/{name}.pipe")
+    if already_exists:
+        f = Path(already_exists[0])
     else:
-        pathname = "pipes"
+        if is_copy(content):
+            pathname = "copies"
+        elif is_materialization(content):
+            pathname = "materializations"
+        elif is_sink(content):
+            pathname = "sinks"
+        elif is_endpoint(content):
+            pathname = "endpoints"
+        else:
+            pathname = "pipes"
 
-    base = Path(folder) / pathname
-    if not base.exists():
-        base.mkdir()
-    f = base / (f"{name}.pipe")
+        base = Path(folder) / pathname
+        if not base.exists():
+            base.mkdir()
+        f = base / (f"{name}.pipe")
     with open(f"{f}", "w") as file:
         file.write(content)
     click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
@@ -357,10 +370,14 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
 
 
 def generate_connection_file(name: str, content: str, folder: str, skip_feedback: bool = False) -> Path:
-    base = Path(folder) / "connections"
-    if not base.exists():
-        base.mkdir()
-    f = base / (f"{name}.connection")
+    already_exists = glob.glob(f"{folder}/**/{name}.connection")
+    if already_exists:
+        f = Path(already_exists[0])
+    else:
+        base = Path(folder) / "connections"
+        if not base.exists():
+            base.mkdir()
+        f = base / (f"{name}.connection")
     with open(f"{f}", "w") as file:
         file.write(content)
     if not skip_feedback:

tinybird/tb/modules/datafile/common.py

@@ -846,6 +846,15 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     lineno=line,
                     pos=pos,
                 )
+            if detected_type in ("Int", "UInt"):
+                t = detected_type
+                raise SchemaSyntaxError(
+                    message=f"Precision is mandatory for {t} types",
+                    hint=f"Hint: use one of {t}8, {t}16, {t}32, {t}64, {t}128, {t}256",
+                    lineno=line,
+                    pos=pos - len(t),
+                )
+
             try:
                 # Imported in the body to be compatible with the CLI
                 from chtoolset.query import check_compatible_types

tinybird/tb/modules/deprecations.py

@@ -9,6 +9,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def branch(args) -> None:
@@ -30,6 +31,7 @@ def branch(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def check(args) -> None:
@@ -47,6 +49,7 @@ def check(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def diff(args) -> None:
@@ -64,6 +67,7 @@ def diff(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def fmt(args) -> None:
@@ -81,6 +85,7 @@ def fmt(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def init(args) -> None:
@@ -98,6 +103,7 @@ def init(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def push(args) -> None:
@@ -115,6 +121,7 @@ def push(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def tag(args) -> None:

tinybird/tb/modules/local_common.py

@@ -18,14 +18,14 @@ TB_LOCAL_HOST = re.sub(r"^https?://", "", os.getenv("TB_LOCAL_HOST", "localhost"
 TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
 
 
-async def get_tinybird_local_client(config_obj: Dict[str, Any], staging: bool = False) -> TinyB:
+async def get_tinybird_local_client(config_obj: Dict[str, Any], test: bool = False, staging: bool = False) -> TinyB:
     """Get a Tinybird client connected to the local environment."""
 
-    config = await get_tinybird_local_config(config_obj)
+    config = await get_tinybird_local_config(config_obj, test=test)
     return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
 
 
-async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
+async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False) -> CLIConfig:
     """Craft a client config with a workspace name based on the path of the project files
 
     It uses the tokens from tinybird local
@@ -47,7 +47,10 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
     # Create a new workspace if path is provided. This is used to isolate the build in a different workspace.
     if path:
         user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
-        ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
+        if test:
+            ws_name = get_test_workspace_name(path)
+        else:
+            ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
         if not ws_name:
             raise AuthNoTokenException()
 
@@ -61,6 +64,13 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
 
         ws = next((ws for ws in local_workspaces if ws["name"] == ws_name), None)
 
+        # If we are running a test, we need to delete the workspace if it already exists
+        if test and ws:
+            requests.delete(
+                f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
+            )
+            ws = None
+
         if not ws:
             await user_client.create_workspace(
                 ws_name, template=None, assign_to_organization_id=user_org_id, version="v1"
@@ -71,7 +81,6 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
                 raise AuthNoTokenException()
 
         ws_token = ws["token"]
-
         config.set_token(ws_token)
         config.set_token_for_host(TB_LOCAL_ADDRESS, ws_token)
         config.set_host(TB_LOCAL_ADDRESS)
@@ -86,3 +95,8 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
 def get_build_workspace_name(path: str) -> str:
     folder_hash = hashlib.sha256(path.encode()).hexdigest()
     return f"Tinybird_Local_Build_{folder_hash}"
+
+
+def get_test_workspace_name(path: str) -> str:
+    folder_hash = hashlib.sha256(path.encode()).hexdigest()
+    return f"Tinybird_Local_Test_{folder_hash}"

tinybird/tb/modules/mock.py

@@ -1,5 +1,6 @@
 import glob
 from pathlib import Path
+from typing import Any, Dict, List
 
 import click
 
@@ -65,41 +66,23 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
 
         datasource_content = datasource_path.read_text()
         config = CLIConfig.get_project_config()
-        user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
         user_token = ctx_config.get("user_token")
 
         if not user_token:
             raise Exception("This action requires authentication. Run 'tb login' first.")
 
-        llm = LLM(user_token=user_token, host=user_client.host)
-        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
-        sql = ""
-        attempts = 0
-        data = []
-        error = ""
-        sql_path = None
-        while True:
-            try:
-                response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
-                sql = extract_xml(response, "sql")
-                sql_path = persist_fixture_sql(datasource_name, sql, folder)
-                sql_format = "JSON" if format_ == "ndjson" else "CSV"
-                result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
-                if sql_format == "JSON":
-                    data = result.get("data", [])[:rows]
-                    error_response = result.get("error", None)
-                    if error_response:
-                        raise Exception(error_response)
-                else:
-                    data = result
-                break
-            except Exception as e:
-                error = str(e)
-                attempts += 1
-                if attempts > 5:
-                    raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
-                else:
-                    continue
+        data = await create_mock_data(
+            datasource_name,
+            datasource_content,
+            rows,
+            prompt,
+            config,
+            ctx_config,
+            user_token,
+            tb_client,
+            format_,
+            folder,
+        )
 
         fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
         click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
@@ -125,3 +108,48 @@ async def append_fixture(
         concurrency=1,
         silent=True,
     )
+
+
+async def create_mock_data(
+    datasource_name: str,
+    datasource_content: str,
+    rows: int,
+    prompt: str,
+    config: CLIConfig,
+    ctx_config: Dict[str, Any],
+    user_token: str,
+    tb_client: TinyB,
+    format_: str,
+    folder: str,
+) -> List[Dict[str, Any]]:
+    user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
+    llm = LLM(user_token=user_token, host=user_client.host)
+    prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
+    sql = ""
+    attempts = 0
+    data = []
+    error = ""
+    sql_path = None
+    while True:
+        try:
+            response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
+            sql = extract_xml(response, "sql")
+            sql_path = persist_fixture_sql(datasource_name, sql, folder)
+            sql_format = "JSON" if format_ == "ndjson" else "CSV"
+            result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
+            if sql_format == "JSON":
+                data = result.get("data", [])[:rows]
+                error_response = result.get("error", None)
+                if error_response:
+                    raise Exception(error_response)
+            else:
+                data = result
+            break
+        except Exception as e:
+            error = str(e)
+            attempts += 1
+            if attempts > 5:
+                raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
+            else:
+                continue
+    return data

tinybird/tb/modules/project.py

@@ -55,6 +55,15 @@ class Project:
     def connections(self) -> List[str]:
         return sorted([Path(f).stem for f in glob.glob(f"{self.path}/**/*.connection", recursive=False)])
 
+    def get_datasource_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.datasource", recursive=False)
+
+    def get_pipe_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.pipe", recursive=False)
+
+    def get_connection_files(self) -> List[str]:
+        return glob.glob(f"{self.path}/**/*.connection", recursive=False)
+
     def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
         try:
             return parse_pipe(filename).datafile

tinybird/tb/modules/test.py

@@ -8,22 +8,25 @@ import difflib
 import glob
 import sys
 import urllib.parse
+from copy import deepcopy
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 import click
+import requests
 import yaml
 from requests import Response
 
 from tinybird.prompts import test_create_prompt
 from tinybird.tb.client import TinyB
-from tinybird.tb.modules.build import process
+from tinybird.tb.modules.build import process as build_project
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.exceptions import CLITestException
+from tinybird.tb.modules.exceptions import CLILocalException, CLITestException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
+from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_test_workspace_name
 from tinybird.tb.modules.project import Project
 
 yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str  # type: ignore[attr-defined]
@@ -77,7 +80,7 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
         project: Project = ctx.ensure_object(dict)["project"]
         client: TinyB = ctx.ensure_object(dict)["client"]
         click.echo(FeedbackManager.highlight(message="\n» Building project"))
-        process(project=project, tb_client=client, watch=False, silent=True)
+        build_project(project=project, tb_client=client, watch=False, silent=True)
         click.echo(FeedbackManager.info(message="✓ Done!\n"))
         config = CLIConfig.get_project_config()
         folder = project.folder
@@ -140,6 +143,8 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
     except Exception as e:
         raise CLITestException(FeedbackManager.error(message=str(e)))
+    finally:
+        cleanup_test_workspace(client, project.folder)
 
 
 @test.command(
@@ -154,7 +159,7 @@ def test_update(ctx: click.Context, pipe: str) -> None:
         project: Project = ctx.ensure_object(dict)["project"]
         folder = project.folder
         click.echo(FeedbackManager.highlight(message="\n» Building project"))
-        process(project=project, tb_client=client, watch=False, silent=True)
+        build_project(project=project, tb_client=client, watch=False, silent=True)
         click.echo(FeedbackManager.info(message="✓ Done!"))
         pipe_tests_path = get_pipe_path(pipe, folder)
         pipe_name = pipe_tests_path.stem
@@ -191,6 +196,8 @@ def test_update(ctx: click.Context, pipe: str) -> None:
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
     except Exception as e:
         raise CLITestException(FeedbackManager.error(message=str(e)))
+    finally:
+        cleanup_test_workspace(client, project.folder)
 
 
 @test.command(
@@ -204,7 +211,7 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
         client: TinyB = ctx.ensure_object(dict)["client"]
         project: Project = ctx.ensure_object(dict)["project"]
         click.echo(FeedbackManager.highlight(message="\n» Building project"))
-        process(project=project, tb_client=client, watch=False, silent=True)
+        build_project(project=project, tb_client=client, watch=False, silent=True)
         click.echo(FeedbackManager.info(message="✓ Done!"))
 
         click.echo(FeedbackManager.highlight(message="\n» Running tests"))
@@ -268,6 +275,8 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
             click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
     except Exception as e:
         raise CLITestException(FeedbackManager.error(message=str(e)))
+    finally:
+        cleanup_test_workspace(client, project.folder)
 
 
 async def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
@@ -301,3 +310,22 @@ def get_pipe_path(name_or_filename: str, folder: str) -> Path:
         raise Exception(f"Pipe {name_or_filename} not found")
 
     return pipe_path
+
+
+def cleanup_test_workspace(client: TinyB, path: str) -> None:
+    user_client = deepcopy(client)
+    try:
+        # ruff: noqa: ASYNC210
+        tokens = requests.get(f"{TB_LOCAL_ADDRESS}/tokens").json()
+    except Exception:
+        raise CLILocalException(
+            FeedbackManager.error(message="Tinybird local is not running. Please run `tb local start` first.")
+        )
+    try:
+        user_token = tokens["user_token"]
+        user_client.token = user_token
+        asyncio.run(
+            user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
+        )
+    except Exception:
+        pass

{tinybird-0.0.1.dev148.dist-info → tinybird-0.0.1.dev152.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev148
+Version: 0.0.1.dev152
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev148.dist-info → tinybird-0.0.1.dev152.dist-info}/RECORD

@@ -3,7 +3,7 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=a_ZhFX2zcB7vRknIcmHKMdQbb0c7TqlTBQ_5hPuWh88,69267
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/prompts.py,sha256=6J-dE0gOSOaIwdEwJUhEQS1Ztllz_qZnpnrv-tDoJWA,35511
+tinybird/prompts.py,sha256=yLXzHB7_P0zjriIvI9UsfnIRwmcwLSEA1tcJMzYJqjw,36456
 tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
 tinybird/sql_template.py,sha256=mK0yeRFctbXTAu0VNMjIzoFBJoh9PoniAVgEatA5SG4,99832
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
@@ -12,7 +12,7 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=BZuPM7MFS7vaEKK5tOMR2bwSAgJudPrJt27uVEwZmTY,40512
-tinybird/tb/__cli__.py,sha256=ND-paC6OJCufE_OWkvnADMob08wWCEmSeglAIwgaUro,252
+tinybird/tb/__cli__.py,sha256=GgehmhHmo8VV_VXA3EVctUEaD_qi0Fd6UJAECARm81o,252
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=uPV6pvi4aYVfaiGs0DQO-eoi1g9dHlrgvhutkXkdJko,1075
 tinybird/tb/client.py,sha256=aaPKq5C77e72kR7IMv9WrvnvNki8mKMOTi9EsCp0eUc,55962
@@ -20,15 +20,15 @@ tinybird/tb/config.py,sha256=jT9xndpeCY_g0HdB5qE2EquC0TFRRnkPnQFWZWd04jo,3998
 tinybird/tb/modules/auth.py,sha256=_OeYnmTH83lnqCgQEdS6K0bx1KBUeRmZk2M7JnRmWpk,9037
 tinybird/tb/modules/build.py,sha256=5SX59ssq0KaRrgzk21KmYUnB6xOdQ_O59n7rxFBQQ1Y,17393
 tinybird/tb/modules/cicd.py,sha256=Cn1DyE28w2maQgIJF4IjZU7J7ULPvMG11DjN2OH-cqE,7161
-tinybird/tb/modules/cli.py,sha256=uXGkK0lrbUMzLRtRuE77khwl0nmfnE6TQuuQFVy8Ug0,14075
+tinybird/tb/modules/cli.py,sha256=AJooz6uiluC7eMO2ZlXluWL-NNLwTKb9kqkv3yqg6S4,14227
 tinybird/tb/modules/common.py,sha256=WzF_zRtAl3FKqbzK6yi_IjeeGIjQNu6SKnP-PX-o3Kk,85196
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
 tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
-tinybird/tb/modules/create.py,sha256=lj7vugRfTo6jNb02sJ9wmV6Y0ASGEbxVAd1FrmaVr58,16943
+tinybird/tb/modules/create.py,sha256=OHUvuHuvP0iecPPGI4eVOHOgR20qy7a_Sw7sbJKuG8g,17411
 tinybird/tb/modules/datasource.py,sha256=V314rkpdVxVMjsp5qcSCTqDlmp4Vu--qM07BoWh-aqs,17783
 tinybird/tb/modules/deployment.py,sha256=T0lwhAT8FwZZcRnA-JDMfPUrj5MOXcQ8blxYIK2igbc,24363
-tinybird/tb/modules/deprecations.py,sha256=kugrWKWLnUL0qHXghObmzEvNIPs5F5O-kTXtYHjaSrI,3632
+tinybird/tb/modules/deprecations.py,sha256=nI_VPCB0Seyhlk2Foomoj3oUnx6GV_8Q5DMYVDi9L-o,3751
 tinybird/tb/modules/endpoint.py,sha256=XySDt3pk66vxOZ0egUfz4bY8bEk3BjOXkv-L0OIJ3sc,12083
 tinybird/tb/modules/exceptions.py,sha256=5jK91w1LPmtqIUfDpHe_Op5OxGz8-p1BPgtLREMIni0,5217
 tinybird/tb/modules/feedback_manager.py,sha256=NoflMdTcUs6JF2IUZS8TatItxWDNCfwaIJnKtbY8e3w,77447
@@ -37,20 +37,20 @@ tinybird/tb/modules/job.py,sha256=n4dSSBgnA8NqD7srGahf2xRj6wxkmX9Vl0J-QJ_a2w0,29
 tinybird/tb/modules/llm.py,sha256=KfsCYmKeW1VQz0iDZhGKCRkQv_Y3kTHh6JuxvofOguE,1076
 tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
 tinybird/tb/modules/local.py,sha256=kxdF8NXZYYwtUwlFHxHKq_PA9CpMrG-rXOg_i_hlvnY,12158
-tinybird/tb/modules/local_common.py,sha256=3nxGcjdxA5Uq0PakhbkiPDVzd66OUwPNW4gNqyYPCe8,3381
+tinybird/tb/modules/local_common.py,sha256=jNbVG_nk5sNduPo3JDSNtop1H2X1Tdd5j4q3xXUR6W8,3965
 tinybird/tb/modules/login.py,sha256=fmXPSdvJnKPv03chptGuu3_Fm6LhP6kUsUKhrmT8rJc,8269
 tinybird/tb/modules/logout.py,sha256=ULooy1cDBD02-r7voZmhV7udA0ML5tVuflJyShrh56Y,1022
 tinybird/tb/modules/materialization.py,sha256=QJX5kCPhhm6IXBO1JsalVfbQdypCe_eOUDZ_WHJZWS8,5478
-tinybird/tb/modules/mock.py,sha256=tyhs6izqVGmDzMHdW5yHiprNPjeJDa2I-cLwrLRDC_U,4816
+tinybird/tb/modules/mock.py,sha256=z_mYNVAu5aeq-5qGB0kSkliOX4j62fgUuLi1-ZcTamA,5304
 tinybird/tb/modules/open.py,sha256=OuctINN77oexpSjth9uoIZPCelKO4Li-yyVxeSnk1io,1371
 tinybird/tb/modules/pipe.py,sha256=AQKEDagO6e3psPVjJkS_MDbn8aK-apAiLp26k7jgAV0,2432
-tinybird/tb/modules/project.py,sha256=RdVOzJiuFPqkCzPhzGetbgcFeiMEn9IMviFCG58XKgI,3142
+tinybird/tb/modules/project.py,sha256=vjw9RB3XcRWqj0izWbYSiTKqzP2n9QdIDXTu8QoRcK0,3502
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/secret.py,sha256=WsqzxxLh9W_jkuHL2JofMXdIJy0lT5WEI-7bQSIDgAc,2921
 tinybird/tb/modules/shell.py,sha256=Zd_4Ak_5tKVX-cw6B4ag36xZeEGHeh-jZpAsIXkoMoE,14116
 tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
 tinybird/tb/modules/telemetry.py,sha256=IHHks8-jnrkyQIY_Y6ZmU8hdVHMGSS0pkSUOgCPcYY8,11346
-tinybird/tb/modules/test.py,sha256=WmPoZBSHBORiM66xp54KxfY3s0YwjFMN_ZTT8xObFO4,12377
+tinybird/tb/modules/test.py,sha256=Yopg89cRwOQpgRzsb9nvu2Z-UR2as2vBjVa5PF3uiK0,13420
 tinybird/tb/modules/token.py,sha256=2fmKwu10_M0pqs6YmJVeILR9ZQB0ejRAET86agASbKM,13488
 tinybird/tb/modules/watch.py,sha256=_92co0BjTikQuy7MbHr4TDu9A75QdHsrAO8v7DlYLg4,8898
 tinybird/tb/modules/workspace.py,sha256=SlK08psp0tu5t_URHyIczm696buW6KD6FPs9Lg1aNRE,6614
@@ -59,7 +59,7 @@ tinybird/tb/modules/datafile/build.py,sha256=d_h3pRFDPFrDKGhpFx2iejY25GuB2k8yfNo
 tinybird/tb/modules/datafile/build_common.py,sha256=LU24kAQmxDJIyoIapDaYG-SU3P4FrMG9UBf8m9PgVSI,4565
 tinybird/tb/modules/datafile/build_datasource.py,sha256=nXEQ0qHdq2ai7jJTv8H2d7eeDPBYzLn8VY7zMtOYb8M,17382
 tinybird/tb/modules/datafile/build_pipe.py,sha256=6Cwjf3BKEF3-oQ9PipsQfK-Z43nSwtA4qJAUoysI7Uc,11385
-tinybird/tb/modules/datafile/common.py,sha256=NIqXpsj7u8uaeSKcUZM-85Kxg35pd08DVAdbVuHmvTU,89400
+tinybird/tb/modules/datafile/common.py,sha256=1xM8R2NVmk8uQpU910Iqp_YGM1SHYoXaLZgjBYIJl54,89771
 tinybird/tb/modules/datafile/diff.py,sha256=MTmj53RYjER4neLgWVjabn-FKVFgh8h8uYiBo55lFQg,6757
 tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/tb/modules/datafile/fixture.py,sha256=DrRWivcvo_1rn7LlVUnHcXccdgx9yVj63mzBkUwCzk8,1420
@@ -79,8 +79,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev148.dist-info/METADATA,sha256=a78Hx_MhAQknWEIA9ss3ykmRIw40G6o3cXiu2bGbXSE,1612
-tinybird-0.0.1.dev148.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev148.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev148.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev148.dist-info/RECORD,,
+tinybird-0.0.1.dev152.dist-info/METADATA,sha256=vHFPvsfjueEX_1WXokHN42243Qf31_7kRfUZqrwS2IA,1612
+tinybird-0.0.1.dev152.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev152.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev152.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev152.dist-info/RECORD,,