tinybird 0.0.1.dev292__tar.gz → 0.0.1.dev293__tar.gz

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev292
+Version: 0.0.1.dev293
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/tinybird/datafile/parse_pipe.py

@@ -11,7 +11,7 @@ from tinybird.datafile.common import (
     parse,
 )
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
-from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
+from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
@@ -54,7 +54,7 @@ def parse_pipe(
             if sql.strip()[0] == "%":
                 secrets_list: Optional[List[str]] = None
                 if secrets:
-                    secrets_list = [f"{secret_template_key(secret)}" for secret in secrets.keys()]
+                    secrets_list = list(secrets.keys())
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/tinybird/sql_template.py

@@ -1405,8 +1405,11 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     namespace = {}
     template_execution_results = TemplateExecutionResults()
     for key in kwargs.get("tb_secrets", []):
+        # Avoid double-prefixing if the key already has the tb_secret_ prefix
         if is_secret_template_key(key):
             template_execution_results.add_template_param(key)
+        else:
+            template_execution_results.add_template_param(secret_template_key(key))
 
     if TB_SECRET_IN_TEST_MODE in kwargs:
         template_execution_results[TB_SECRET_IN_TEST_MODE] = None
@@ -1415,15 +1418,20 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
         try:
             key = secret_template_key(x)
             if key in template_execution_results.template_params:
+                # secret available: Always use workspace secret regardless of test mode
                 template_execution_results.add_ch_param(x)
                 return Symbol("{" + sqlescape(x) + ": String}")
             else:
+                # secret not available: Check test mode and defaults
                 is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
-                if is_test_mode and default is None:
-                    return Symbol("{" + sqlescape(x) + ": String}")
-                elif default is not None:
+                if default is not None:
+                    # Use provided default value
                     return default
+                elif is_test_mode:
+                    # In test mode without default - return placeholder
+                    return Symbol("{" + sqlescape(x) + ": String}")
                 else:
+                    # Not in test mode, no secret, no default - raise error
                     raise SQLTemplateException(
                         f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
                     )

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev292'
-__revision__ = '78dba1d'
+__version__ = '0.0.1.dev293'
+__revision__ = '8c3de05'

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/tinybird/tb/modules/build.py

@@ -3,7 +3,7 @@ import time
 from copy import deepcopy
 from functools import partial
 from pathlib import Path
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 from urllib.parse import urlencode
 
 import click
@@ -19,6 +19,7 @@ from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.playground import folder_playground
 from tinybird.tb.modules.dev_server import BuildStatus, start_server
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_local_tokens
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell, print_table_formatted
 from tinybird.tb.modules.watch import watch_files, watch_project
@@ -34,6 +35,7 @@ def build(ctx: click.Context, watch: bool) -> None:
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     if obj["env"] == "cloud":
         raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
 
@@ -46,6 +48,12 @@ def build(ctx: click.Context, watch: bool) -> None:
             )
         )
 
+    # First, build vendored workspaces if present
+
+    build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
+    # Ensure SHARED_WITH workspaces exist before building and sharing
+    build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=False)
     if watch:
@@ -65,6 +73,7 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
         return dev_cloud(ctx)
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     build_status = BuildStatus()
     if ui:
         server_thread = threading.Thread(
@@ -74,6 +83,11 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
         # Wait for the server to start
         time.sleep(0.5)
 
+    # Build vendored workspaces before starting dev build/watch
+    build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
+    # Ensure SHARED_WITH workspaces exist before dev build/watch
+    build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=True, build_status=build_status)
     run_watch(
@@ -127,6 +141,134 @@ def check_filenames(filenames: List[str]):
         parser(filename)
 
 
+def find_workspace_or_create(user_client: TinyB, workspace_name: str) -> Optional[str]:
+    # Get a client scoped to the vendored workspace using the user token
+    ws_token = None
+    org_id = None
+    try:
+        # Fetch org id and workspaces with tokens
+        info = user_client.user_workspaces_with_organization(version="v1")
+        org_id = info.get("organization_id")
+        workspaces = info.get("workspaces", [])
+        found = next((w for w in workspaces if w.get("name") == workspace_name), None)
+        if found:
+            ws_token = found.get("token")
+        # If still not found, try the generic listing
+        if not ws_token:
+            workspaces_full = user_client.user_workspaces_and_branches(version="v1")
+            created_ws = next(
+                (w for w in workspaces_full.get("workspaces", []) if w.get("name") == workspace_name), None
+            )
+            if created_ws:
+                ws_token = created_ws.get("token")
+    except Exception:
+        ws_token = None
+
+    # If workspace doesn't exist, try to create it and fetch its token
+    if not ws_token:
+        try:
+            user_client.create_workspace(workspace_name, assign_to_organization_id=org_id, version="v1")
+            # Fetch token for newly created workspace
+            info_after = user_client.user_workspaces_and_branches(version="v1")
+            created = next((w for w in info_after.get("workspaces", []) if w.get("name") == workspace_name), None)
+            ws_token = created.get("token") if created else None
+        except Exception as e:
+            click.echo(
+                FeedbackManager.warning(
+                    message=(f"Skipping vendored workspace '{workspace_name}': unable to create or resolve token ({e})")
+                )
+            )
+
+    return ws_token
+
+
+def build_vendored_workspaces(project: Project, tb_client: TinyB, config: Dict[str, Any]) -> None:
+    """Build each vendored workspace under project.vendor_path if present.
+
+    Directory structure expected: vendor/<workspace_name>/<data_project_inside>
+    Each top-level directory under vendor is treated as a separate workspace
+    whose project files will be built using that workspace's token.
+    """
+    try:
+        vendor_root = Path(project.vendor_path)
+
+        if not vendor_root.exists() or not vendor_root.is_dir():
+            return
+
+        tokens = get_local_tokens()
+        user_token = tokens["user_token"]
+        user_client = deepcopy(tb_client)
+        user_client.token = user_token
+
+        # Iterate over vendored workspace folders
+        for ws_dir in sorted([p for p in vendor_root.iterdir() if p.is_dir()]):
+            workspace_name = ws_dir.name
+            ws_token = find_workspace_or_create(user_client, workspace_name)
+
+            if not ws_token:
+                click.echo(
+                    FeedbackManager.warning(
+                        message=f"Skipping vendored workspace '{workspace_name}': could not resolve token after creation"
+                    )
+                )
+                continue
+
+            # Build using a client scoped to the vendor workspace token
+            vendor_client = deepcopy(tb_client)
+            vendor_client.token = ws_token
+            vendor_project = Project(folder=str(ws_dir), workspace_name=workspace_name, max_depth=project.max_depth)
+
+            # Do not exit on error to allow main project to continue
+            process(
+                project=vendor_project,
+                tb_client=vendor_client,
+                watch=False,
+                silent=False,
+                exit_on_error=True,
+                load_fixtures=True,
+            )
+    except Exception as e:
+        # Never break the main build due to vendored build errors
+        click.echo(FeedbackManager.error_exception(error=e))
+
+
+def build_shared_with_workspaces(project: Project, tb_client: TinyB, config: Dict[str, Any]) -> None:
+    """Scan project for .datasource files and ensure SHARED_WITH workspaces exist."""
+
+    try:
+        # Gather SHARED_WITH workspace names from all .datasource files
+        datasource_files = project.get_datasource_files()
+        shared_ws_names = set()
+
+        for filename in datasource_files:
+            try:
+                doc = parse_datasource(filename).datafile
+                for ws_name in doc.shared_with or []:
+                    shared_ws_names.add(ws_name)
+            except Exception:
+                # Ignore parse errors here; they'll be handled during the main process()
+                continue
+
+        if not shared_ws_names:
+            return
+
+        # Need a user token to list/create workspaces
+        tokens = get_local_tokens()
+        user_token = tokens.get("user_token")
+        if not user_token:
+            click.echo(FeedbackManager.info_skipping_shared_with_entry())
+            return
+
+        user_client = deepcopy(tb_client)
+        user_client.token = user_token
+
+        # Ensure each SHARED_WITH workspace exists
+        for ws_name in sorted(shared_ws_names):
+            find_workspace_or_create(user_client, ws_name)
+    except Exception as e:
+        click.echo(FeedbackManager.error_exception(error=e))
+
+
 def dev_cloud(
     ctx: click.Context,
 ) -> None:

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/tinybird/tb/modules/feedback_manager.py

@@ -1209,7 +1209,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)
 
-    highlight_building_project = info_highlight_message("\n» Building project...")
+    highlight_building_project = info_highlight_message("» Building project...")
 
     success = success_message("{message}")
     info = info_message("{message}")

{tinybird-0.0.1.dev292 → tinybird-0.0.1.dev293}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev292
+Version: 0.0.1.dev293
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird