tinybird 0.0.1.dev291__tar.gz → 0.0.1.dev293__tar.gz

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev291
+Version: 0.0.1.dev293
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/context.py

@@ -3,10 +3,11 @@ from typing import TYPE_CHECKING
 
 # Avoid circular import error
 if TYPE_CHECKING:
-    from tinybird.user import User
+    from hfi.hfi_workspace_data import HfiWorkspaceData
+
 
 workspace_id: ContextVar[str] = ContextVar("workspace_id")
-workspace: ContextVar["User"] = ContextVar("workspace")
+hfi_workspace_data: ContextVar["HfiWorkspaceData"] = ContextVar("hfi_workspace_data")
 table_id: ContextVar[str] = ContextVar("table_id")
 hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/datafile/parse_pipe.py

@@ -11,7 +11,7 @@ from tinybird.datafile.common import (
     parse,
 )
 from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
-from tinybird.sql_template import get_template_and_variables, render_sql_template, secret_template_key
+from tinybird.sql_template import get_template_and_variables, render_sql_template
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tornado_template import UnClosedIfError
 
@@ -54,7 +54,7 @@ def parse_pipe(
             if sql.strip()[0] == "%":
                 secrets_list: Optional[List[str]] = None
                 if secrets:
-                    secrets_list = [f"{secret_template_key(secret)}" for secret in secrets.keys()]
+                    secrets_list = list(secrets.keys())
                 # Setting test_mode=True to ignore errors on required parameters and
                 # secrets_in_test_mode=False to raise errors on missing secrets
                 sql, _, variable_warnings = render_sql_template(

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/sql_template.py

@@ -1405,8 +1405,11 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     namespace = {}
     template_execution_results = TemplateExecutionResults()
     for key in kwargs.get("tb_secrets", []):
+        # Avoid double-prefixing if the key already has the tb_secret_ prefix
         if is_secret_template_key(key):
             template_execution_results.add_template_param(key)
+        else:
+            template_execution_results.add_template_param(secret_template_key(key))
 
     if TB_SECRET_IN_TEST_MODE in kwargs:
         template_execution_results[TB_SECRET_IN_TEST_MODE] = None
@@ -1415,15 +1418,20 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
         try:
             key = secret_template_key(x)
             if key in template_execution_results.template_params:
+                # secret available: Always use workspace secret regardless of test mode
                 template_execution_results.add_ch_param(x)
                 return Symbol("{" + sqlescape(x) + ": String}")
             else:
+                # secret not available: Check test mode and defaults
                 is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
-                if is_test_mode and default is None:
-                    return Symbol("{" + sqlescape(x) + ": String}")
-                elif default is not None:
+                if default is not None:
+                    # Use provided default value
                     return default
+                elif is_test_mode:
+                    # In test mode without default - return placeholder
+                    return Symbol("{" + sqlescape(x) + ": String}")
                 else:
+                    # Not in test mode, no secret, no default - raise error
                     raise SQLTemplateException(
                         f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
                     )

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev291'
-__revision__ = '87da698'
+__version__ = '0.0.1.dev293'
+__revision__ = '8c3de05'

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/agent/agent.py

@@ -34,10 +34,12 @@ from tinybird.tb.modules.agent.mock_agent import MockAgent
 from tinybird.tb.modules.agent.models import create_model
 from tinybird.tb.modules.agent.prompts import (
     agent_system_prompt,
+    fixtures_prompt,
     load_custom_project_rules,
     resources_prompt,
     secrets_prompt,
     service_datasources_prompt,
+    vendor_files_prompt,
 )
 from tinybird.tb.modules.agent.testing_agent import TestingAgent
 from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
@@ -322,6 +324,14 @@ class TinybirdAgent:
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_vendor_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return vendor_files_prompt(self.project)
+
+        @self.agent.instructions
+        def get_fixture_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return fixtures_prompt(self.project)
+
         @self.agent.instructions
         def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
             return service_datasources_prompt()

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/agent/explore_agent.py

@@ -12,6 +12,7 @@ from tinybird.tb.modules.agent.prompts import (
     resources_prompt,
     service_datasources_prompt,
     tone_and_style_instructions,
+    vendor_files_prompt,
 )
 from tinybird.tb.modules.agent.tools.diff_resource import diff_resource
 from tinybird.tb.modules.agent.tools.execute_query import execute_query
@@ -76,6 +77,10 @@ Once you finish the task, return a valid response for the task to complete.
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_vendor_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return vendor_files_prompt(self.project)
+
         @self.agent.instructions
         def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
             return service_datasources_prompt()

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/agent/mock_agent.py

@@ -6,7 +6,7 @@ from pydantic_ai.usage import Usage
 
 from tinybird.tb.modules.agent.animations import ThinkingAnimation
 from tinybird.tb.modules.agent.models import create_model
-from tinybird.tb.modules.agent.prompts import resources_prompt
+from tinybird.tb.modules.agent.prompts import fixtures_prompt, resources_prompt
 from tinybird.tb.modules.agent.tools.mock import generate_mock_fixture
 from tinybird.tb.modules.agent.utils import TinybirdAgentContext
 from tinybird.tb.modules.project import Project
@@ -194,6 +194,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_fixture_files(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return fixtures_prompt(self.project)
+
     def run(self, task: str, deps: TinybirdAgentContext, usage: Usage):
         result = self.agent.run_sync(
             task,

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/agent/prompts.py

@@ -35,8 +35,10 @@ available_commands = [
     "`tb sink ls`: List all sinks",
     "`tb workspace current`: Show the current workspace",
     "`tb workspace clear --yes`: Delete all resources in the workspace (Only available in Tinybird Local)",
+    "`tb workspace ls`: List all workspaces",
     "`tb local start --skip-new-version`: Start Tinybird Local container in non-interactive mode",
     "`tb local restart --skip-new-version --yes`: Restart Tinybird Local container in non-interactive mode",
+    "`tb pull --only-vendored`: Pull only the vendored datasources from other workspaces",
 ]
 
 plan_instructions = """
@@ -166,7 +168,6 @@ SQL >
 
 def resources_prompt(project: Project) -> str:
     files = project.get_project_files()
-    fixture_files = project.get_fixture_files()
 
     resources_content = "# Existing resources in the project:\n"
     if files:
@@ -184,7 +185,36 @@ def resources_prompt(project: Project) -> str:
     else:
         resources_content += "No resources found"
 
+    return resources_content
+
+
+def vendor_files_prompt(project: Project) -> str:
+    files = project.get_vendored_files()
+    content = "# Datasources shared from other workspaces:\n"
+    if files:
+        resources: list[dict[str, Any]] = []
+        for filename in files:
+            file_path = Path(filename)
+            workspace_name = file_path.parent.parent.name
+            resource = {
+                "path": str(file_path.relative_to(project.folder)),
+                "type": get_resource_type(file_path),
+                "name": f"{workspace_name}.{file_path.stem}",
+                "content": file_path.read_text(),
+                "origin_workspace": workspace_name,
+            }
+            resources.append(resource)
+        content += format_as_xml(resources, root_tag="resources", item_tag="resource")
+    else:
+        content += "No datasources shared from other workspaces"
+
+    return content
+
+
+def fixtures_prompt(project: Project) -> str:
+    fixture_files = project.get_fixture_files()
     fixture_content = "# Fixture files in the project:\n"
+
     if fixture_files:
         fixtures: list[dict[str, Any]] = []
         for filename in fixture_files:
@@ -199,7 +229,7 @@ def resources_prompt(project: Project) -> str:
     else:
         fixture_content += "No fixture files found"
 
-    return resources_content + "\n" + fixture_content
+    return fixture_content
 
 
 def service_datasources_prompt() -> str:
@@ -954,6 +984,23 @@ After changes have been deployed and promoted, if you want to deploy other chang
 If after running a deployment, the error contains a recommended forward query, use it to update the .datasource file.
 </dev_notes>
 
+## Sharing datasources with other workspaces
+To share a Data Source, in the .datasource file you want to share, add the destination workspace(s). For example:
+
+```
+SHARED_WITH >
+    destination_workspace,
+    other_destination_workspace
+```
+
+## Working with shared datasources:
+
+The following limitations apply to shared datasources:
+- Shared datasources are read-only.
+- You can't share a shared datasource, only the original.
+- You can't check the quarantine of a shared datasource.
+- You can't create a Materialized View from a shared datasource.
+
 # Working with any type of pipe file:
 {pipe_instructions}
 {pipe_example}

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/agent/tools/run_command.py

@@ -3,6 +3,7 @@ import subprocess
 import click
 from pydantic_ai import RunContext
 
+from tinybird.tb.modules.agent.prompts import available_commands
 from tinybird.tb.modules.agent.utils import (
     AgentRunCancelled,
     SubAgentRunCancelled,
@@ -20,6 +21,14 @@ def run_command(ctx: RunContext[TinybirdAgentContext], command: str):
         command (str): The command to run. Required. Examples: `tb --local sql "select 1"`, `tb --cloud datasource ls`, `tb --help`
     """
     try:
+        clean_commands = [cmd.split(":")[0].replace("`", "").split("[")[0] for cmd in available_commands]
+        available_commands_to_cloud = [f"tb --cloud {command.replace('tb ', '')}" for command in clean_commands]
+        available_commands_to_local = [f"tb --local {command.replace('tb ', '')}" for command in clean_commands]
+        all_commands = clean_commands + available_commands_to_cloud + available_commands_to_local
+
+        if not any(cmd.startswith(command) for cmd in all_commands):
+            raise SubAgentRunCancelled(f"Command {command} not found in the list of available commands")
+
         ctx.deps.thinking_animation.stop()
         force_confirmation = " deploy" in command.lower() or " truncate" in command.lower()
         confirmation = show_confirmation(

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/build.py

@@ -3,7 +3,7 @@ import time
 from copy import deepcopy
 from functools import partial
 from pathlib import Path
-from typing import Any, Callable, Dict, List
+from typing import Any, Callable, Dict, List, Optional
 from urllib.parse import urlencode
 
 import click
@@ -19,6 +19,7 @@ from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.playground import folder_playground
 from tinybird.tb.modules.dev_server import BuildStatus, start_server
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_local_tokens
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.shell import Shell, print_table_formatted
 from tinybird.tb.modules.watch import watch_files, watch_project
@@ -34,6 +35,7 @@ def build(ctx: click.Context, watch: bool) -> None:
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     if obj["env"] == "cloud":
         raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
 
@@ -46,6 +48,12 @@ def build(ctx: click.Context, watch: bool) -> None:
             )
         )
 
+    # First, build vendored workspaces if present
+
+    build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
+    # Ensure SHARED_WITH workspaces exist before building and sharing
+    build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=False)
     if watch:
@@ -65,6 +73,7 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
         return dev_cloud(ctx)
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
     build_status = BuildStatus()
     if ui:
         server_thread = threading.Thread(
@@ -74,6 +83,11 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
         # Wait for the server to start
         time.sleep(0.5)
 
+    # Build vendored workspaces before starting dev build/watch
+    build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
+    # Ensure SHARED_WITH workspaces exist before dev build/watch
+    build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
+
     click.echo(FeedbackManager.highlight_building_project())
     process(project=project, tb_client=tb_client, watch=True, build_status=build_status)
     run_watch(
@@ -127,6 +141,134 @@ def check_filenames(filenames: List[str]):
         parser(filename)
 
 
+def find_workspace_or_create(user_client: TinyB, workspace_name: str) -> Optional[str]:
+    # Get a client scoped to the vendored workspace using the user token
+    ws_token = None
+    org_id = None
+    try:
+        # Fetch org id and workspaces with tokens
+        info = user_client.user_workspaces_with_organization(version="v1")
+        org_id = info.get("organization_id")
+        workspaces = info.get("workspaces", [])
+        found = next((w for w in workspaces if w.get("name") == workspace_name), None)
+        if found:
+            ws_token = found.get("token")
+        # If still not found, try the generic listing
+        if not ws_token:
+            workspaces_full = user_client.user_workspaces_and_branches(version="v1")
+            created_ws = next(
+                (w for w in workspaces_full.get("workspaces", []) if w.get("name") == workspace_name), None
+            )
+            if created_ws:
+                ws_token = created_ws.get("token")
+    except Exception:
+        ws_token = None
+
+    # If workspace doesn't exist, try to create it and fetch its token
+    if not ws_token:
+        try:
+            user_client.create_workspace(workspace_name, assign_to_organization_id=org_id, version="v1")
+            # Fetch token for newly created workspace
+            info_after = user_client.user_workspaces_and_branches(version="v1")
+            created = next((w for w in info_after.get("workspaces", []) if w.get("name") == workspace_name), None)
+            ws_token = created.get("token") if created else None
+        except Exception as e:
+            click.echo(
+                FeedbackManager.warning(
+                    message=(f"Skipping vendored workspace '{workspace_name}': unable to create or resolve token ({e})")
+                )
+            )
+
+    return ws_token
+
+
+def build_vendored_workspaces(project: Project, tb_client: TinyB, config: Dict[str, Any]) -> None:
+    """Build each vendored workspace under project.vendor_path if present.
+
+    Directory structure expected: vendor/<workspace_name>/<data_project_inside>
+    Each top-level directory under vendor is treated as a separate workspace
+    whose project files will be built using that workspace's token.
+    """
+    try:
+        vendor_root = Path(project.vendor_path)
+
+        if not vendor_root.exists() or not vendor_root.is_dir():
+            return
+
+        tokens = get_local_tokens()
+        user_token = tokens["user_token"]
+        user_client = deepcopy(tb_client)
+        user_client.token = user_token
+
+        # Iterate over vendored workspace folders
+        for ws_dir in sorted([p for p in vendor_root.iterdir() if p.is_dir()]):
+            workspace_name = ws_dir.name
+            ws_token = find_workspace_or_create(user_client, workspace_name)
+
+            if not ws_token:
+                click.echo(
+                    FeedbackManager.warning(
+                        message=f"Skipping vendored workspace '{workspace_name}': could not resolve token after creation"
+                    )
+                )
+                continue
+
+            # Build using a client scoped to the vendor workspace token
+            vendor_client = deepcopy(tb_client)
+            vendor_client.token = ws_token
+            vendor_project = Project(folder=str(ws_dir), workspace_name=workspace_name, max_depth=project.max_depth)
+
+            # Do not exit on error to allow main project to continue
+            process(
+                project=vendor_project,
+                tb_client=vendor_client,
+                watch=False,
+                silent=False,
+                exit_on_error=True,
+                load_fixtures=True,
+            )
+    except Exception as e:
+        # Never break the main build due to vendored build errors
+        click.echo(FeedbackManager.error_exception(error=e))
+
+
+def build_shared_with_workspaces(project: Project, tb_client: TinyB, config: Dict[str, Any]) -> None:
+    """Scan project for .datasource files and ensure SHARED_WITH workspaces exist."""
+
+    try:
+        # Gather SHARED_WITH workspace names from all .datasource files
+        datasource_files = project.get_datasource_files()
+        shared_ws_names = set()
+
+        for filename in datasource_files:
+            try:
+                doc = parse_datasource(filename).datafile
+                for ws_name in doc.shared_with or []:
+                    shared_ws_names.add(ws_name)
+            except Exception:
+                # Ignore parse errors here; they'll be handled during the main process()
+                continue
+
+        if not shared_ws_names:
+            return
+
+        # Need a user token to list/create workspaces
+        tokens = get_local_tokens()
+        user_token = tokens.get("user_token")
+        if not user_token:
+            click.echo(FeedbackManager.info_skipping_shared_with_entry())
+            return
+
+        user_client = deepcopy(tb_client)
+        user_client.token = user_token
+
+        # Ensure each SHARED_WITH workspace exists
+        for ws_name in sorted(shared_ws_names):
+            find_workspace_or_create(user_client, ws_name)
+    except Exception as e:
+        click.echo(FeedbackManager.error_exception(error=e))
+
+
 def dev_cloud(
     ctx: click.Context,
 ) -> None:

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird/tb/modules/feedback_manager.py

@@ -1209,7 +1209,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
 
     debug_running_file = print_message("** Running {file}", bcolors.CGREY)
 
-    highlight_building_project = info_highlight_message("\n» Building project...")
+    highlight_building_project = info_highlight_message("» Building project...")
 
     success = success_message("{message}")
     info = info_message("{message}")

{tinybird-0.0.1.dev291 → tinybird-0.0.1.dev293}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev291
+Version: 0.0.1.dev293
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird