tinybird 0.0.1.dev148__tar.gz → 0.0.1.dev151__tar.gz

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev148
+Version: 0.0.1.dev151
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/prompts.py

@@ -401,7 +401,7 @@ Follow the instructions and generate the following response with no additional t
 def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
     feedback_history = ""
     if feedback and history:
-        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
 it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
 Use the following feedback and history to regenerate the response:
 Feedback to improve the response:
@@ -420,7 +420,9 @@ You are a Tinybird expert. You will be given a prompt to generate new or update
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 
 {feedback_history}
 
@@ -443,14 +445,16 @@ Use the following format to generate the response and do not wrap it in any othe
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
         feedback_history=feedback_history,
     )
 
 
 def mock_prompt(rows: int, feedback: str = "") -> str:
     if feedback:
-        feedback = f"""In case the <feedback> tag is present and not empty, 
+        feedback = f"""In case the <feedback> tag is present and not empty,
 it means there was a previous attempt to generate the resources and the system provided feedback about the previous response.
 Use the following feedback to regenerate the response:
 <feedback>{feedback}</feedback>
@@ -515,7 +519,7 @@ SCHEMA >
     `customer_id` UInt64 `json:$.customer_id`,
     `order_date` DateTime `json:$.order_date`,
     `total_amount` Float64 `json:$.total_amount`,
-    `items` Array(String) `json:$.items[:]` // This is an array field 
+    `items` Array(String) `json:$.items[:]` // This is an array field
 
 ### Desired final output of the query:
 {{
@@ -680,19 +684,34 @@ connection_instructions = """
     - Content cannot be empty.
     - The connection names must be unique.
     - No indentation is allowed for property names
-    - We only support kafka connections for now
+    - We support kafka, gcs and s3 connections for now
 </connection_file_instructions>
 """
 
-connection_example = """
-<connection_content>
+kafka_connection_example = """
+<kafka_connection_content>
 TYPE kafka
 KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }}
 KAFKA_SECURITY_PROTOCOL SASL_SSL
 KAFKA_SASL_MECHANISM PLAIN
 KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }}
 KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }}
-</connection_content>
+</kafka_connection_content>
+"""
+
+gcs_connection_example = """
+<gcs_connection_content>
+TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON", "") }}
+</gcs_connection_content>
+"""
+
+s3_connection_example = """
+<s3_connection_content>
+TYPE s3
+S3_REGION {{ tb_secret("PRODUCTION_S3_REGION", "") }}
+S3_ARN {{ tb_secret("PRODUCTION_S3_ARN", "") }}
+</s3_connection_content>
 """
 
 datasource_instructions = """
@@ -703,6 +722,7 @@ datasource_instructions = """
     - Use MergeTree engine by default.
     - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
+    - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
 </datasource_file_instructions>
 """
 
@@ -775,7 +795,7 @@ sql_instructions = """
     {{%if not defined(end_date)%}}
     AND now()
     {{%else%}}
-    AND {{{{DateTime(end_date)}}}} 
+    AND {{{{DateTime(end_date)}}}}
     {{%end%}}
     </valid_condition_without_now>
     - Parameters must not be quoted.
@@ -862,7 +882,9 @@ Follow these instructions when creating or updating .pipe files:
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
 {connection_instructions}
-{connection_example}
+{kafka_connection_example}
+{gcs_connection_example}
+{s3_connection_example}
 </pipe_file_instructions>
 <test_file_instructions>
 Follow these instructions when creating or updating .yaml files for tests:
@@ -884,7 +906,9 @@ Follow these instructions when evolving a datasource schema:
         test_instructions=test_instructions,
         deployment_instructions=deployment_instructions,
         connection_instructions=connection_instructions,
-        connection_example=connection_example,
+        kafka_connection_example=kafka_connection_example,
+        gcs_connection_example=gcs_connection_example,
+        s3_connection_example=s3_connection_example,
     )
 
 
@@ -950,7 +974,7 @@ SCHEMA >
     `payload` String `json:$.payload`
 
 FORWARD_QUERY >
-    select timestamp, toUUID(session_id) as session_id, action, version, payload 
+    select timestamp, toUUID(session_id) as session_id, action, version, payload
     </forward_query_example>
 </deployment_instruction>
 """
@@ -968,7 +992,7 @@ Current README.md file:
 - If it is not present in the current readme, generate a new ## Tinybird section with the following content:
     - ### Overview section:
         - Explaining the purpose of the project.
-    - ### Data sources section: 
+    - ### Data sources section:
         - Explaining the purpose of each datasource.
         - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
         curl -X POST "{host}/v0/events?name=events" \

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev148'
-__revision__ = '7b2789f'
+__version__ = '0.0.1.dev151'
+__revision__ = 'd9da445'

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/modules/cli.py

@@ -323,16 +323,23 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, s
         return None
 
     commands_always_cloud = ["pull", "infra"]
-    commands_always_local = ["build", "test", "dev"]
-    if (cloud or command in commands_always_cloud) and command not in commands_always_local:
+    commands_always_local = ["build", "dev"]
+    command_always_test = ["test"]
+
+    if (
+        (cloud or command in commands_always_cloud)
+        and command not in commands_always_local
+        and command not in command_always_test
+    ):
         click.echo(
             FeedbackManager.gray(message=f"Running against Tinybird Cloud: Workspace {config.get('name', 'default')}")
         )
         return _get_tb_client(config.get("token", None), config["host"], staging=staging)
     local = command in commands_always_local
+    test = command in command_always_test
     if not local and command not in commands_always_local and command:
         click.echo(FeedbackManager.gray(message="Running against Tinybird Local"))
-    return await get_tinybird_local_client(config, staging=staging)
+    return await get_tinybird_local_client(config, test=test, staging=staging)
 
 
 def get_target_env(cloud: bool) -> str:

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/modules/create.py

@@ -240,6 +240,11 @@ TYPE ENDPOINT
         pipes_paths = [
             Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
         ]
+        connections_paths = [
+            Path(folder) / "connections" / f
+            for f in os.listdir(Path(folder) / "connections")
+            if f.endswith(".connection")
+        ]
         resources_xml = "\n".join(
             [
                 f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -254,6 +259,14 @@ TYPE ENDPOINT
                     )
                     for pipe in pipes_paths
                 ]
+                + [
+                    (
+                        "connection",
+                        conn.stem,
+                        conn.read_text(),
+                    )
+                    for conn in connections_paths
+                ]
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/modules/datafile/common.py

@@ -846,6 +846,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                     lineno=line,
                     pos=pos,
                 )
+
             try:
                 # Imported in the body to be compatible with the CLI
                 from chtoolset.query import check_compatible_types

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/modules/deprecations.py

@@ -9,6 +9,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def branch(args) -> None:
@@ -30,6 +31,7 @@ def branch(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def check(args) -> None:
@@ -47,6 +49,7 @@ def check(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def diff(args) -> None:
@@ -64,6 +67,7 @@ def diff(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def fmt(args) -> None:
@@ -81,6 +85,7 @@ def fmt(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def init(args) -> None:
@@ -98,6 +103,7 @@ def init(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def push(args) -> None:
@@ -115,6 +121,7 @@ def push(args) -> None:
     context_settings=dict(
         ignore_unknown_options=True,
     ),
+    hidden=True,
 )
 @click.argument("args", nargs=-1, type=click.UNPROCESSED)
 def tag(args) -> None:

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/modules/local_common.py

@@ -18,14 +18,14 @@ TB_LOCAL_HOST = re.sub(r"^https?://", "", os.getenv("TB_LOCAL_HOST", "localhost"
 TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
 
 
-async def get_tinybird_local_client(config_obj: Dict[str, Any], staging: bool = False) -> TinyB:
+async def get_tinybird_local_client(config_obj: Dict[str, Any], test: bool = False, staging: bool = False) -> TinyB:
     """Get a Tinybird client connected to the local environment."""
 
-    config = await get_tinybird_local_config(config_obj)
+    config = await get_tinybird_local_config(config_obj, test=test)
     return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
 
 
-async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
+async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False) -> CLIConfig:
     """Craft a client config with a workspace name based on the path of the project files
 
     It uses the tokens from tinybird local
@@ -47,7 +47,10 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
     # Create a new workspace if path is provided. This is used to isolate the build in a different workspace.
     if path:
         user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
-        ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
+        if test:
+            ws_name = get_test_workspace_name(path)
+        else:
+            ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
         if not ws_name:
             raise AuthNoTokenException()
 
@@ -61,6 +64,13 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
 
         ws = next((ws for ws in local_workspaces if ws["name"] == ws_name), None)
 
+        # If we are running a test, we need to delete the workspace if it already exists
+        if test and ws:
+            requests.delete(
+                f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
+            )
+            ws = None
+
         if not ws:
             await user_client.create_workspace(
                 ws_name, template=None, assign_to_organization_id=user_org_id, version="v1"
@@ -71,7 +81,6 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
                 raise AuthNoTokenException()
 
         ws_token = ws["token"]
-
         config.set_token(ws_token)
         config.set_token_for_host(TB_LOCAL_ADDRESS, ws_token)
         config.set_host(TB_LOCAL_ADDRESS)
@@ -86,3 +95,8 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
 def get_build_workspace_name(path: str) -> str:
     folder_hash = hashlib.sha256(path.encode()).hexdigest()
     return f"Tinybird_Local_Build_{folder_hash}"
+
+
+def get_test_workspace_name(path: str) -> str:
+    folder_hash = hashlib.sha256(path.encode()).hexdigest()
+    return f"Tinybird_Local_Test_{folder_hash}"

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird/tb/modules/test.py

@@ -8,22 +8,25 @@ import difflib
 import glob
 import sys
 import urllib.parse
+from copy import deepcopy
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
 import click
+import requests
 import yaml
 from requests import Response
 
 from tinybird.prompts import test_create_prompt
 from tinybird.tb.client import TinyB
-from tinybird.tb.modules.build import process
+from tinybird.tb.modules.build import process as build_project
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.config import CLIConfig
-from tinybird.tb.modules.exceptions import CLITestException
+from tinybird.tb.modules.exceptions import CLILocalException, CLITestException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
+from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_test_workspace_name
 from tinybird.tb.modules.project import Project
 
 yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str  # type: ignore[attr-defined]
@@ -77,7 +80,7 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
         project: Project = ctx.ensure_object(dict)["project"]
         client: TinyB = ctx.ensure_object(dict)["client"]
         click.echo(FeedbackManager.highlight(message="\n» Building project"))
-        process(project=project, tb_client=client, watch=False, silent=True)
+        build_project(project=project, tb_client=client, watch=False, silent=True)
         click.echo(FeedbackManager.info(message="✓ Done!\n"))
         config = CLIConfig.get_project_config()
         folder = project.folder
@@ -140,6 +143,8 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
     except Exception as e:
         raise CLITestException(FeedbackManager.error(message=str(e)))
+    finally:
+        cleanup_test_workspace(client, project.folder)
 
 
 @test.command(
@@ -154,7 +159,7 @@ def test_update(ctx: click.Context, pipe: str) -> None:
         project: Project = ctx.ensure_object(dict)["project"]
         folder = project.folder
         click.echo(FeedbackManager.highlight(message="\n» Building project"))
-        process(project=project, tb_client=client, watch=False, silent=True)
+        build_project(project=project, tb_client=client, watch=False, silent=True)
         click.echo(FeedbackManager.info(message="✓ Done!"))
         pipe_tests_path = get_pipe_path(pipe, folder)
         pipe_name = pipe_tests_path.stem
@@ -191,6 +196,8 @@ def test_update(ctx: click.Context, pipe: str) -> None:
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
     except Exception as e:
         raise CLITestException(FeedbackManager.error(message=str(e)))
+    finally:
+        cleanup_test_workspace(client, project.folder)
 
 
 @test.command(
@@ -204,7 +211,7 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
         client: TinyB = ctx.ensure_object(dict)["client"]
         project: Project = ctx.ensure_object(dict)["project"]
         click.echo(FeedbackManager.highlight(message="\n» Building project"))
-        process(project=project, tb_client=client, watch=False, silent=True)
+        build_project(project=project, tb_client=client, watch=False, silent=True)
         click.echo(FeedbackManager.info(message="✓ Done!"))
 
         click.echo(FeedbackManager.highlight(message="\n» Running tests"))
@@ -268,6 +275,8 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
             click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
     except Exception as e:
         raise CLITestException(FeedbackManager.error(message=str(e)))
+    finally:
+        cleanup_test_workspace(client, project.folder)
 
 
 async def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
@@ -301,3 +310,22 @@ def get_pipe_path(name_or_filename: str, folder: str) -> Path:
         raise Exception(f"Pipe {name_or_filename} not found")
 
     return pipe_path
+
+
+def cleanup_test_workspace(client: TinyB, path: str) -> None:
+    user_client = deepcopy(client)
+    try:
+        # ruff: noqa: ASYNC210
+        tokens = requests.get(f"{TB_LOCAL_ADDRESS}/tokens").json()
+    except Exception:
+        raise CLILocalException(
+            FeedbackManager.error(message="Tinybird local is not running. Please run `tb local start` first.")
+        )
+    try:
+        user_token = tokens["user_token"]
+        user_client.token = user_token
+        asyncio.run(
+            user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
+        )
+    except Exception:
+        pass

{tinybird-0.0.1.dev148 → tinybird-0.0.1.dev151}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev148
+Version: 0.0.1.dev151
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird