tinybird 0.0.1.dev285__py3-none-any.whl → 0.0.1.dev287__py3-none-any.whl

tinybird/sql_template.py

@@ -1411,7 +1411,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     if TB_SECRET_IN_TEST_MODE in kwargs:
         template_execution_results[TB_SECRET_IN_TEST_MODE] = None
 
-    def set_tb_secret(x):
+    def set_tb_secret(x, default=None):
         try:
             key = secret_template_key(x)
             if key in template_execution_results.template_params:
@@ -1419,8 +1419,10 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
                 return Symbol("{" + sqlescape(x) + ": String}")
             else:
                 is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
-                if is_test_mode:
+                if is_test_mode and default is None:
                     return Symbol("{" + sqlescape(x) + ": String}")
+                elif default is not None:
+                    return default
                 else:
                     raise SQLTemplateException(
                         f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
@@ -2266,6 +2268,18 @@ def render_sql_template(
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'test'. Check the secret exists in the Workspace and the token has the required scope.
+    >>> render_sql_template("select * from table where str = {{tb_secret('test', 'default_value')}}", secrets = [])
+    ("select * from table where str = 'default_value'", {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test', 'default_value')}}", secrets = [ 'tb_secret_test' ])
+    ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test', '')}}")
+    ("select * from table where str = ''", {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test', 'default_value')}}", test_mode=True)
+    ("select * from table where str = 'default_value'", {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test', '')}}", test_mode=True)
+    ("select * from table where str = ''", {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_secret('test', 'default_value')}}", secrets = [ 'tb_secret_test' ], test_mode=True)
+    ('select * from table where str = {test: String}', {}, [])
     >>> render_sql_template("select * from table where str = {{String(test)}} and category = {{String(category, 'shirts')}} and color = {{ Int32(color)}}", test_mode=False)
     Traceback (most recent call last):
     ...

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev285'
-__revision__ = '0a30a28'
+__version__ = '0.0.1.dev287'
+__revision__ = 'f35d4e1'

tinybird/tb/modules/agent/agent.py

@@ -35,6 +35,7 @@ from tinybird.tb.modules.agent.prompts import (
     load_custom_project_rules,
     resources_prompt,
     secrets_prompt,
+    service_datasources_prompt,
 )
 from tinybird.tb.modules.agent.testing_agent import TestingAgent
 from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
@@ -285,6 +286,10 @@ class TinybirdAgent:
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return service_datasources_prompt()
+
         @self.agent.instructions
         def get_secrets(ctx: RunContext[TinybirdAgentContext]) -> str:
             return secrets_prompt(self.project)
@@ -293,14 +298,12 @@ class TinybirdAgent:
         self.messages.append(message)
 
     def _build_agent_deps(self, config: dict[str, Any], run_id: Optional[str] = None) -> TinybirdAgentContext:
-        client = TinyB(token=self.token, host=self.host)
         project = self.project
         folder = self.project.folder
         local_client = get_tinybird_local_client(config, test=False, silent=False)
         test_client = get_tinybird_local_client(config, test=True, silent=True)
         return TinybirdAgentContext(
             # context does not support the whole client, so we need to pass only the functions we need
-            explore_data=client.explore_data,
             build_project=partial(build_project, project=project, config=config),
             deploy_project=partial(deploy_project, project=project, config=config),
             deploy_check_project=partial(deploy_check_project, project=project, config=config),

tinybird/tb/modules/agent/explore_agent.py

@@ -10,6 +10,7 @@ from tinybird.tb.modules.agent.models import create_model
 from tinybird.tb.modules.agent.prompts import (
     explore_data_instructions,
     resources_prompt,
+    service_datasources_prompt,
     tone_and_style_instructions,
 )
 from tinybird.tb.modules.agent.tools.diff_resource import diff_resource
@@ -75,6 +76,10 @@ Once you finish the task, return a valid response for the task to complete.
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_service_datasources(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return service_datasources_prompt()
+
     def run(self, task: str, deps: TinybirdAgentContext, usage: Usage):
         result = self.agent.run_sync(
             task,

tinybird/tb/modules/agent/prompts.py

@@ -12,6 +12,7 @@ from tinybird.prompts import (
     pipe_instructions,
     sink_pipe_instructions,
 )
+from tinybird.service_datasources import get_organization_service_datasources, get_tinybird_service_datasources
 from tinybird.tb.modules.project import Project
 
 available_commands = [
@@ -118,11 +119,6 @@ sql_instructions = """
     - When you use defined function with a paremeter inside, do NOT add quotes around the parameter:
     <invalid_defined_function_with_parameter>{% if defined('my_param') %}</invalid_defined_function_with_parameter>
     <valid_defined_function_without_parameter>{% if defined(my_param) %}</valid_defined_function_without_parameter>
-    - Use datasource names as table names when doing SELECT statements.
-    - Do not use pipe names as table names.
-    - The available datasource names to use in the SQL are the ones present in the existing_resources section or the ones you will create.
-    - Use node names as table names only when nodes are present in the same file.
-    - Do not reference the current node name in the SQL.
     - SQL queries only accept SELECT statements with conditions, aggregations, joins, etc.
     - ONLY SELECT statements are allowed in any sql query.
     - When using functions try always ClickHouse functions first, then SQL functions.
@@ -133,7 +129,7 @@ sql_instructions = """
 datafile_instructions = """
 <datafile_instructions>
 - Endpoint files will be created under the `/endpoints` folder.
-- Materialized pipe files will be created under the `/materialized` folder.
+- Materialized pipe files will be created under the `/materializations` folder.
 - Sink pipe files will be created under the `/sinks` folder.
 - Copy pipe files will be created under the `/copies` folder.
 - Connection files will be created under the `/connections` folder.
@@ -184,7 +180,7 @@ def resources_prompt(project: Project) -> str:
                 "content": file_path.read_text(),
             }
             resources.append(resource)
-        resources_content = format_as_xml(resources, root_tag="resources", item_tag="resource")
+        resources_content += format_as_xml(resources, root_tag="resources", item_tag="resource")
     else:
         resources_content += "No resources found"
 
@@ -198,7 +194,7 @@ def resources_prompt(project: Project) -> str:
                 "name": file_path.stem,
             }
             fixtures.append(fixture)
-        fixture_content = format_as_xml(fixtures, root_tag="fixtures", item_tag="fixture")
+        fixture_content += format_as_xml(fixtures, root_tag="fixtures", item_tag="fixture")
 
     else:
         fixture_content += "No fixture files found"
@@ -206,6 +202,50 @@ def resources_prompt(project: Project) -> str:
     return resources_content + "\n" + fixture_content
 
 
+def service_datasources_prompt() -> str:
+    def build_content(ds: dict[str, Any]) -> str:
+        content = "DESCRIPTION >\n"
+        content += f"  {ds.get('description', 'No description')}\n"
+
+        content += "SCHEMA >\n"
+        for column in ds.get("columns", []):
+            content += f"  `{column.get('name', '')}` {column.get('type', '')}\n"
+
+        if engine := ds.get("engine", {}).get("engine", ""):
+            content += f"ENGINE {engine}\n"
+        if sorting_key := ds.get("engine", {}).get("sorting_key", ""):
+            content += f"ENGINE_SORTING_KEY {sorting_key}\n"
+        if partition_key := ds.get("engine", {}).get("partition_key", ""):
+            content += f"ENGINE_PARTITION_KEY {partition_key}\n"
+
+        return content
+
+    skip_datasources = ["tinybird.bi_stats", "tinybird.bi_stats_rt", "tinybird.releases_log", "tinybird.hook_log"]
+    service_datasources = [
+        {"name": ds["name"], "content": build_content(ds)}
+        for ds in get_tinybird_service_datasources()
+        if ds["name"] not in skip_datasources
+    ]
+    content = "# Service datasources:\n"
+    content += format_as_xml(
+        service_datasources, root_tag="workspace_service_datasources", item_tag="service_datasource"
+    )
+    content += "\n#Organization service datasources:\n"
+    skip_datasources = ["organization.bi_stats", "organization.bi_stats_rt"]
+    org_service_datasources = [
+        {"name": ds["name"], "content": build_content(ds)}
+        for ds in get_organization_service_datasources()
+        if ds["name"] not in skip_datasources
+    ]
+    content += format_as_xml(
+        org_service_datasources,
+        root_tag="organization_service_datasources",
+        item_tag="service_datasource",
+    )
+
+    return content
+
+
 def secrets_prompt(project: Project) -> str:
     """Generate a prompt showing available secrets from .env.local file."""
     secrets = project.get_secrets()
@@ -224,7 +264,7 @@ def secrets_prompt(project: Project) -> str:
         secrets_list.append(secret)
 
     if secrets_list:
-        secrets_content = format_as_xml(secrets_list, root_tag="secrets", item_tag="secret")
+        secrets_content += format_as_xml(secrets_list, root_tag="secrets", item_tag="secret")
 
     return secrets_content
 
@@ -245,7 +285,7 @@ def tests_files_prompt(project: Project) -> str:
                 "content": file_path.read_text(),
             }
             resources.append(resource)
-        resources_content = format_as_xml(resources, root_tag="resources", item_tag="resource")
+        resources_content += format_as_xml(resources, root_tag="resources", item_tag="resource")
     else:
         resources_content += "No resources found"
 
@@ -260,7 +300,7 @@ def tests_files_prompt(project: Project) -> str:
                 "content": file_path.read_text(),
             }
             tests.append(test)
-        test_content = format_as_xml(tests, root_tag="tests", item_tag="test")
+        test_content += format_as_xml(tests, root_tag="tests", item_tag="test")
     else:
         test_content += "No test files found"
 
@@ -957,6 +997,14 @@ They can be run on a schedule, or executed on demand.
 {sql_agent_instructions}
 {sql_instructions}
 
+## Referencing tables in SQL queries:
+The following resources can be used as tables in SQL queries:
+- Datasources (.datasource files)
+- Materialized views (.datasource files target of .pipe files with `TYPE MATERIALIZED` defined)
+- Endpoints (.pipe files with `TYPE ENDPOINT` defined)
+- Default pipes (.pipe files with no `TYPE` defined)
+- Node names present in the same .pipe file
+
 {secrets_instructions}
 
 {external_tables_instructions}

tinybird/tb/modules/agent/utils.py

@@ -40,7 +40,6 @@ class TinybirdAgentContext(BaseModel):
     workspace_name: str
     thinking_animation: Any
     get_project_files: Callable[[], List[str]]
-    explore_data: Callable[[str], str]
     build_project: Callable[..., None]
     build_project_test: Callable[..., None]
     deploy_project: Callable[..., None]

tinybird/tb/modules/workspace.py

@@ -157,7 +157,7 @@ def create_workspace(
         )
 
     config = CLIConfig.get_project_config()
-    user_token = config.get_user_token()
+    user_token = get_user_token(config)
     fork = False
 
     if not user_token:

{tinybird-0.0.1.dev285.dist-info → tinybird-0.0.1.dev287.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev285
+Version: 0.0.1.dev287
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -21,6 +21,7 @@ Requires-Dist: GitPython~=3.1.32
 Requires-Dist: humanfriendly~=8.2
 Requires-Dist: plotext==5.3.2
 Requires-Dist: prompt_toolkit==3.0.48
+Requires-Dist: logfire-api==4.2.0
 Requires-Dist: pydantic~=2.11.7
 Requires-Dist: pydantic-ai-slim[anthropic]~=0.5.0
 Requires-Dist: pydantic-ai-slim[retries]~=0.5.0

{tinybird-0.0.1.dev285.dist-info → tinybird-0.0.1.dev287.dist-info}/RECORD

@@ -4,20 +4,21 @@ tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=XY8d83pRlq-LH7xHMApkaEebfXEWLjDzrGe1prpcTHE,69778
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
 tinybird/prompts.py,sha256=HoDv9TxPiP8v2XoGTWYxP133dK9CEbXVv4XE5IT339c,45483
+tinybird/service_datasources.py,sha256=o6Az3T6OvcChR_7GXRu7sJH173KzGg1Gv-dNd0bI_vY,46085
 tinybird/sql.py,sha256=UZJLop6zA9tTPEaS-Fq7M-QyzmC5uV_tIeXZzkjnhso,48299
-tinybird/sql_template.py,sha256=OTsBBtGnVL4oUrEEFK19Mi5GHm-BP7v-zViXfPvYqJs,100774
+tinybird/sql_template.py,sha256=kaF5pi-f2JiWSYXEF8JsU1OIxvdu2homHnw4MYjq0n8,101953
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
 tinybird/sql_toolset.py,sha256=19WPr4S3SR--Iw-VPgmDnLmhOZyHhTxnj3-_Yq3OgEU,19767
 tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=v5-nkXHUhysu4i9Z4WVv0-sBbh6xSYUH5q5xHSY2xTI,4194
 tinybird/ch_utils/engine.py,sha256=4X1B-iuhdW_mxKnX_m3iCsxgP9RPVgR75g7yH1vsJ6A,40851
-tinybird/datafile/common.py,sha256=uRSQnSS3yv1iuJRZHgenphe-g1TEjZr04ICcl7QYcZw,105521
+tinybird/datafile/common.py,sha256=euJAKSu6GA67oIcRvrrBGsGI1nZtkSWM45ikjLUlJz0,106049
 tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
 tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=8Avu_3xqfp0OfVOmQW7sJg4dwrhj1I6b2-pLutdFn8w,247
+tinybird/tb/__cli__.py,sha256=nZRxzzjaWGhmlgqsxUdTdtORhtemm6OdxvjRGKVVdbc,247
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
 tinybird/tb/client.py,sha256=IQRaInDjOwr9Fzaz3_xXc3aUGqh94tM2lew7IZbB9eM,53733
@@ -66,21 +67,21 @@ tinybird/tb/modules/test.py,sha256=O2-mS4uMU6nPi7yWPpWzshAgOlYKiGS-tkM12pXQGMI,1
 tinybird/tb/modules/test_common.py,sha256=7EpwGVs3SQTVUOM4fX3iq2fITquYvLtsEz599mN6Eqc,14525
 tinybird/tb/modules/token.py,sha256=ZhW_o7XCr90wJRhMN6816vyo_TVfnzPXyIhrhzQ7oZ0,13807
 tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,8819
-tinybird/tb/modules/workspace.py,sha256=USsG8YEXlwf7F2PjTMCuQ2lB8ya-erbv8VywNJYq6mc,11173
+tinybird/tb/modules/workspace.py,sha256=tCP1zZMwBhLRGm22TGfpSd4cHvQLAS1o_azIXv_r6uw,11172
 tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
 tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
-tinybird/tb/modules/agent/agent.py,sha256=38X69i7zL93I30hFFMfl6cAwHDgzIO_PP2ImGJF-UDE,35247
+tinybird/tb/modules/agent/agent.py,sha256=ZoqV5kTwUVoJ4B2pJZSaooAKztEV6CBUoyeSrxLmusI,35341
 tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
 tinybird/tb/modules/agent/banner.py,sha256=l6cO5Fi7lbVKp-GsBP8jf3IkjOWxg2jpAt9NBCy0WR8,4085
 tinybird/tb/modules/agent/command_agent.py,sha256=0Z08rQsir59zQAr-kkOvsKIFpIBsBSTGJJ1VgqqF5WA,3654
 tinybird/tb/modules/agent/compactor.py,sha256=BK5AxZFhrp3xWnsRnYaleiYoIWtVNc-_m650Hsopt8g,13841
-tinybird/tb/modules/agent/explore_agent.py,sha256=W5pp99wixVSyb66qEVwBv8rZWpJ7JgzjJ_sN4d9u4Gg,3903
+tinybird/tb/modules/agent/explore_agent.py,sha256=gyD5uV5TJwV24eeQiSwhkgfNPb4mtbeH7t2qSdoc18U,4100
 tinybird/tb/modules/agent/memory.py,sha256=vBewB_64L_wHoT4tLT6UX2uxcHwSY880QZ26F9rPqXs,3793
 tinybird/tb/modules/agent/mock_agent.py,sha256=zbAZfAqdSLUtMr2VqO0erWpzjT2F1tTcuYjvHb-gvbA,8023
 tinybird/tb/modules/agent/models.py,sha256=eokO8XlY-kVJOsbqiVporGUAOCyKAXCO5xgTEK9SM6Y,2208
-tinybird/tb/modules/agent/prompts.py,sha256=F49-d_nRd7ai4ikizFHx0_NPGEZ46AgRtUUnxqZhPew,42164
+tinybird/tb/modules/agent/prompts.py,sha256=QG_lj0QHRo8oLlJroPEGoG9_fbPN7sRA7e8jlEmmUok,44033
 tinybird/tb/modules/agent/testing_agent.py,sha256=AtwtJViH7805i7djyBgDb7SSUtDyJnw0TWJu6lBFsrg,2953
-tinybird/tb/modules/agent/utils.py,sha256=4jsQCAH2zBx13w20DOBrrDnQq9n2rKG9sGhBkJYiPzs,31744
+tinybird/tb/modules/agent/utils.py,sha256=3v8cZCx4D8rSvziWMCnqdK54yPXd7a2DfACUkxCC5vY,31705
 tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tinybird/tb/modules/agent/tools/analyze.py,sha256=CR5LXg4fou-zYEksqnjpJ0icvxJVoKnTctoI1NRvqCM,3873
 tinybird/tb/modules/agent/tools/append.py,sha256=6uByExBpb9rVzB0tebWyLFbfkjEPSNxIGFbZrJTaGVI,8056
@@ -118,8 +119,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev285.dist-info/METADATA,sha256=TSthFD5eA-D8BSU0WggS9U102dd9NAOg4m8COH4jClc,1811
-tinybird-0.0.1.dev285.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev285.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev285.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev285.dist-info/RECORD,,
+tinybird-0.0.1.dev287.dist-info/METADATA,sha256=7KJm2mX3vjlPwyqJ8Ov6y8cNd-LqFewBwvDxTIXgDgg,1845
+tinybird-0.0.1.dev287.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev287.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev287.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev287.dist-info/RECORD,,