tinybird 0.0.1.dev292__py3-none-any.whl → 0.0.1.dev293__py3-none-any.whl

tinybird/datafile/parse_pipe.py

@@ -11,7 +11,7 @@ from tinybird.datafile.common import (
     parse,
 )
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
-from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
+from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
@@ -54,7 +54,7 @@ def parse_pipe(
             if sql.strip()[0] == "%":
                 secrets_list: Optional[List[str]] = None
                 if secrets:
-                    secrets_list = [f"{secret_template_key(secret)}" for secret in secrets.keys()]
+                    secrets_list = list(secrets.keys())
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(

tinybird/sql_template.py

@@ -1405,8 +1405,11 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     namespace = {}
     template_execution_results = TemplateExecutionResults()
     for key in kwargs.get("tb_secrets", []):
+        # Avoid double-prefixing if the key already has the tb_secret_ prefix
         if is_secret_template_key(key):
             template_execution_results.add_template_param(key)
+        else:
+            template_execution_results.add_template_param(secret_template_key(key))
 
     if TB_SECRET_IN_TEST_MODE in kwargs:
         template_execution_results[TB_SECRET_IN_TEST_MODE] = None
@@ -1415,15 +1418,20 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
         try:
             key = secret_template_key(x)
             if key in template_execution_results.template_params:
+                # secret available: Always use workspace secret regardless of test mode
                 template_execution_results.add_ch_param(x)
                 return Symbol("{" + sqlescape(x) + ": String}")
             else:
+                # secret not available: Check test mode and defaults
                 is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
-                if is_test_mode and default is None:
-                    return Symbol("{" + sqlescape(x) + ": String}")
-                elif default is not None:
+                if default is not None:
+                    # Use provided default value
                     return default
+                elif is_test_mode:
+                    # In test mode without default - return placeholder
+                    return Symbol("{" + sqlescape(x) + ": String}")
                 else:
+                    # Not in test mode, no secret, no default - raise error
                     raise SQLTemplateException(
                         f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
                     )

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev292'
-__revision__ = '78dba1d'
+__version__ = '0.0.1.dev293'
+__revision__ = '8c3de05'

tinybird/tb/modules/build.py

@@ -3,7 +3,7 @@ import time
 from copy import deepcopy
 from functools import partial
 from pathlib import Path
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 from urllib.parse import urlencode
 
 import click
@@ -19,6 +19,7 @@ from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.playground import folder_playground
 from tinybird.tb.modules.dev_server import BuildStatus, start_server
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_local_tokens
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell, print_table_formatted
 from tinybird.tb.modules.watch import watch_files, watch_project
@@ -34,6 +35,7 @@ def build(ctx: click.Context, watch: bool) -> None:
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     if obj["env"] == "cloud":
         raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
 
@@ -46,6 +48,12 @@ def build(ctx: click.Context, watch: bool) -> None:
             )
         )
 
+    # First, build vendored workspaces if present
+
+    build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
+    # Ensure SHARED_WITH workspaces exist before building and sharing
+    build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=False)
     if watch:
@@ -65,6 +73,7 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
         return dev_cloud(ctx)
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     build_status = BuildStatus()
     if ui:
         server_thread = threading.Thread(
@@ -74,6 +83,11 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
         # Wait for the server to start
         time.sleep(0.5)
 
+    # Build vendored workspaces before starting dev build/watch
+    build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
+    # Ensure SHARED_WITH workspaces exist before dev build/watch
+    build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=True, build_status=build_status)
     run_watch(
@@ -127,6 +141,134 @@ def check_filenames(filenames: List[str]):
         parser(filename)
 
 
+def find_workspace_or_create(user_client: TinyB, workspace_name: str) -> Optional[str]:
+    # Get a client scoped to the vendored workspace using the user token
+    ws_token = None
+    org_id = None
+    try:
+        # Fetch org id and workspaces with tokens
+        info = user_client.user_workspaces_with_organization(version="v1")
+        org_id = info.get("organization_id")
+        workspaces = info.get("workspaces", [])
+        found = next((w for w in workspaces if w.get("name") == workspace_name), None)
+        if found:
+            ws_token = found.get("token")
+        # If still not found, try the generic listing
+        if not ws_token:
+            workspaces_full = user_client.user_workspaces_and_branches(version="v1")
+            created_ws = next(
+                (w for w in workspaces_full.get("workspaces", []) if w.get("name") == workspace_name), None
+            )
+            if created_ws:
+                ws_token = created_ws.get("token")
+    except Exception:
+        ws_token = None
+
+    # If workspace doesn't exist, try to create it and fetch its token
+    if not ws_token:
+        try:
+            user_client.create_workspace(workspace_name, assign_to_organization_id=org_id, version="v1")
+            # Fetch token for newly created workspace
+            info_after = user_client.user_workspaces_and_branches(version="v1")
+            created = next((w for w in info_after.get("workspaces", []) if w.get("name") == workspace_name), None)
+            ws_token = created.get("token") if created else None
+        except Exception as e:
+            click.echo(
+                FeedbackManager.warning(
+                    message=(f"Skipping vendored workspace '{workspace_name}': unable to create or resolve token ({e})")
+                )
+            )
+
+    return ws_token
+
+
+def build_vendored_workspaces(project: Project, tb_client: TinyB, config: Dict[str, Any]) -> None:
+    """Build each vendored workspace under project.vendor_path if present.
+
+    Directory structure expected: vendor/<workspace_name>/<data_project_inside>
+    Each top-level directory under vendor is treated as a separate workspace
+    whose project files will be built using that workspace's token.
+    """
+    try:
+        vendor_root = Path(project.vendor_path)
+
+        if not vendor_root.exists() or not vendor_root.is_dir():
+            return
+
+        tokens = get_local_tokens()
+        user_token = tokens["user_token"]
+        user_client = deepcopy(tb_client)
+        user_client.token = user_token
+
+        # Iterate over vendored workspace folders
+        for ws_dir in sorted([p for p in vendor_root.iterdir() if p.is_dir()]):
+            workspace_name = ws_dir.name
+            ws_token = find_workspace_or_create(user_client, workspace_name)
+
+            if not ws_token:
+                click.echo(
+                    FeedbackManager.warning(
+                        message=f"Skipping vendored workspace '{workspace_name}': could not resolve token after creation"
+                    )
+                )
+                continue
+
+            # Build using a client scoped to the vendor workspace token
+            vendor_client = deepcopy(tb_client)
+            vendor_client.token = ws_token
+            vendor_project = Project(folder=str(ws_dir), workspace_name=workspace_name, max_depth=project.max_depth)
+
+            # Do not exit on error to allow main project to continue
+            process(
+                project=vendor_project,
+                tb_client=vendor_client,
+                watch=False,
+                silent=False,
+                exit_on_error=True,
+                load_fixtures=True,
+            )
+    except Exception as e:
+        # Never break the main build due to vendored build errors
+        click.echo(FeedbackManager.error_exception(error=e))
+
+
+def build_shared_with_workspaces(project: Project, tb_client: TinyB, config: Dict[str, Any]) -> None:
+    """Scan project for .datasource files and ensure SHARED_WITH workspaces exist."""
+
+    try:
+        # Gather SHARED_WITH workspace names from all .datasource files
+        datasource_files = project.get_datasource_files()
+        shared_ws_names = set()
+
+        for filename in datasource_files:
+            try:
+                doc = parse_datasource(filename).datafile
+                for ws_name in doc.shared_with or []:
+                    shared_ws_names.add(ws_name)
+            except Exception:
+                # Ignore parse errors here; they'll be handled during the main process()
+                continue
+
+        if not shared_ws_names:
+            return
+
+        # Need a user token to list/create workspaces
+        tokens = get_local_tokens()
+        user_token = tokens.get("user_token")
+        if not user_token:
+            click.echo(FeedbackManager.info_skipping_shared_with_entry())
+            return
+
+        user_client = deepcopy(tb_client)
+        user_client.token = user_token
+
+        # Ensure each SHARED_WITH workspace exists
+        for ws_name in sorted(shared_ws_names):
+            find_workspace_or_create(user_client, ws_name)
+    except Exception as e:
+        click.echo(FeedbackManager.error_exception(error=e))
+
+
 def dev_cloud(
     ctx: click.Context,
 ) -> None:

tinybird/tb/modules/feedback_manager.py

@@ -1209,7 +1209,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)
 
-    highlight_building_project = info_highlight_message("\n» Building project...")
+    highlight_building_project = info_highlight_message("» Building project...")
 
     success = success_message("{message}")
     info = info_message("{message}")

{tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev293.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev292
+Version: 0.0.1.dev293
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev293.dist-info}/RECORD

@@ -6,7 +6,7 @@ tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=HoDv9TxPiP8v2XoGTWYxP133dK9CEbXVv4XE5IT339c,45483
 tinybird/service_datasources.py,sha256=o6Az3T6OvcChR_7GXRu7sJH173KzGg1Gv-dNd0bI_vY,46085
 tinybird/sql.py,sha256=7pkwQMUVy0CH5zFgLMZyCx1BeaJSQYC05EmOb9vO3Ls,48694
-tinybird/sql_template.py,sha256=kaF5pi-f2JiWSYXEF8JsU1OIxvdu2homHnw4MYjq0n8,101953
+tinybird/sql_template.py,sha256=rNePArEmx5GemxW6GJAiNnln0NRWncGgFe0EbC8G2f4,102463
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
 tinybird/sql_toolset.py,sha256=Y2_fQrbk5GSNoRncAmRS_snpV61XQbiOy4uYkeF6pr4,19767
 tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
@@ -17,13 +17,13 @@ tinybird/datafile/common.py,sha256=hr9Qv4Xot21l2dW6xKPa5_JUu8pnGuDNnVpDf8pgthc,1
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
-tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=-6WKEO5Hmmfh0zrBTi4mPaxkqDYeWDApz2022V9XsK8,247
+tinybird/datafile/parse_pipe.py,sha256=8e9LMecSQVWHC4AXf8cdxoQ1nxUR4fTObYxTctO_EXQ,3816
+tinybird/tb/__cli__.py,sha256=pu9JJjmy3RBdLvb1l9ZlYQN2pYAF28tMrRS8qOplXdI,247
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
 tinybird/tb/client.py,sha256=IQRaInDjOwr9Fzaz3_xXc3aUGqh94tM2lew7IZbB9eM,53733
 tinybird/tb/config.py,sha256=mhMTGnMB5KcxGoh3dewIr2Jjsa6pHE183gCPAQWyp6o,3973
-tinybird/tb/modules/build.py,sha256=lSCbmrqDxqa8bG009qgp2nNRpQW7JWgSen7UQNCfLVE,7657
+tinybird/tb/modules/build.py,sha256=u31cE8ao_OU32_gat0fDkVEn5dP6tRwsesr5Tnynd3I,13624
 tinybird/tb/modules/build_common.py,sha256=KGCtyY1JGexXXhcMVFI3e-gRavmRfCoZnfGPSTYHGxk,12962
 tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
 tinybird/tb/modules/cli.py,sha256=-hkwg2ZIWPtldkLkgdY7RYWacOJtkMx58eqqYzMoMfw,17793
@@ -39,7 +39,7 @@ tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XX
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=ksRj6mfDb9Xv63PhTkV_uKSosgysHElqagg3RTt21Do,11958
 tinybird/tb/modules/exceptions.py,sha256=KEj3E-zN7Z28igvptlngXIjlHR9vAJk59Vs4zpwm4vA,5469
-tinybird/tb/modules/feedback_manager.py,sha256=xkZjwGOV50-oc0y9h58ei4MZ1mwOMOaN_kUcGrtNH48,78132
+tinybird/tb/modules/feedback_manager.py,sha256=duigLI2UIeMzP5IOa7XUty23NRvNssm2ZAKicuun3Pg,78130
 tinybird/tb/modules/info.py,sha256=F5vY4kHS_kyO2uSBKac92HoOb447oDeRlzpwtAHTuKc,6872
 tinybird/tb/modules/infra.py,sha256=JE9oLIyF4bi_JBoe-BgZ5HhKp_lQgSihuSV1KIS02Qs,32709
 tinybird/tb/modules/job.py,sha256=wBsnu8UPTOha2rkLvucgmw4xYv73ubmui3eeSIF68ZM,3107
@@ -121,8 +121,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev292.dist-info/METADATA,sha256=d2RNeqvNyqjfXrpEIlGB9lV-Kh2KUu3bD8c99JGeGJQ,1845
-tinybird-0.0.1.dev292.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev292.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev292.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev292.dist-info/RECORD,,
+tinybird-0.0.1.dev293.dist-info/METADATA,sha256=t4Tsiss9S3-uN8On8FaKHdRoWv1uNYRYxwEAutvIY_A,1845
+tinybird-0.0.1.dev293.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev293.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev293.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev293.dist-info/RECORD,,