tinybird 0.0.1.dev269__tar.gz → 0.0.1.dev270__tar.gz

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev269
+Version: 0.0.1.dev270
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev269'
-__revision__ = '24737f5'
+__version__ = '0.0.1.dev270'
+__revision__ = '77ee059'

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird/tb/modules/agent/agent.py

@@ -27,7 +27,12 @@ from tinybird.tb.modules.agent.memory import (
     save_messages,
 )
 from tinybird.tb.modules.agent.models import create_model
-from tinybird.tb.modules.agent.prompts import agent_system_prompt, load_custom_project_rules, resources_prompt
+from tinybird.tb.modules.agent.prompts import (
+    agent_system_prompt,
+    load_custom_project_rules,
+    resources_prompt,
+    secrets_prompt,
+)
 from tinybird.tb.modules.agent.testing_agent import TestingAgent
 from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
 from tinybird.tb.modules.agent.tools.append import append_file, append_url
@@ -40,6 +45,7 @@ from tinybird.tb.modules.agent.tools.get_endpoint_stats import get_endpoint_stat
 from tinybird.tb.modules.agent.tools.get_openapi_definition import get_openapi_definition
 from tinybird.tb.modules.agent.tools.mock import mock
 from tinybird.tb.modules.agent.tools.plan import plan
+from tinybird.tb.modules.agent.tools.secret import create_or_update_secrets
 from tinybird.tb.modules.agent.utils import AgentRunCancelled, TinybirdAgentContext, show_confirmation, show_input
 from tinybird.tb.modules.build_common import process as build_process
 from tinybird.tb.modules.common import _analyze, _get_tb_client, echo_safe_humanfriendly_tables_format_pretty_table
@@ -51,6 +57,7 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.login_common import login
 from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
 from tinybird.tb.modules.project import Project
+from tinybird.tb.modules.secret_common import load_secrets
 from tinybird.tb.modules.test_common import run_tests as run_tests_common
 
 
@@ -108,6 +115,12 @@ class TinybirdAgent:
                     takes_ctx=True,
                 ),
                 Tool(diff_resource, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(
+                    create_or_update_secrets,
+                    docstring_format="google",
+                    require_parameter_descriptions=True,
+                    takes_ctx=True,
+                ),
             ],
             history_processors=[compact_messages],
         )
@@ -207,6 +220,10 @@ class TinybirdAgent:
         def get_project_files(ctx: RunContext[TinybirdAgentContext]) -> str:
             return resources_prompt(self.project)
 
+        @self.agent.instructions
+        def get_secrets(ctx: RunContext[TinybirdAgentContext]) -> str:
+            return secrets_prompt(self.project)
+
     def add_message(self, message: ModelMessage) -> None:
         self.messages.append(message)
 
@@ -251,6 +268,7 @@ class TinybirdAgent:
             local_host=local_client.host,
             local_token=local_client.token,
             run_id=run_id,
+            load_secrets_local=partial(load_secrets, project=project, client=local_client),
         )
 
     def run(self, user_prompt: str, config: dict[str, Any]) -> None:

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird/tb/modules/agent/prompts.py

@@ -5,16 +5,12 @@ from typing import Any
 from pydantic_ai import format_as_xml
 
 from tinybird.prompts import (
-    connection_instructions,
     copy_pipe_instructions,
     datasource_example,
     datasource_instructions,
-    gcs_connection_example,
-    kafka_connection_example,
     materialized_pipe_instructions,
     pipe_example,
     pipe_instructions,
-    s3_connection_example,
     sink_pipe_instructions,
 )
 from tinybird.tb.modules.project import Project
@@ -25,18 +21,20 @@ When asked to create a plan, you MUST respond with this EXACT format and NOTHING
 Plan description: [One sentence describing what will be built]
 
 Steps:
-1. Connection: [name] - [description]
-2. Datasource: [name] - [description] - Depends on: [connection_name (optional)]
-3. Endpoint: [name] - [description] - Depends on: [resources]
-4. Materialized pipe: [name] - [description] - Depends on: [resources]
-5. Materialized datasource: [name] - [description] - Depends on: [resources]
-6. Sink: [name] - [description] - Depends on: [resources]
-7. Copy: [name] - [description] - Depends on: [resources]
-8. Generate mock data: [datasource_name]
-9. Append existing fixture: [fixture_pathname] - Target: [datasource_name]
+1. Create secrets: [key1, key2, ...] - Create all required secrets in .env.local in one step
+2. Connection: [name] - [description]
+3. Datasource: [name] - [description] - Depends on: [connection_name (optional)]
+4. Endpoint: [name] - [description] - Depends on: [resources]
+5. Materialized pipe: [name] - [description] - Depends on: [resources]
+6. Materialized datasource: [name] - [description] - Depends on: [resources]
+7. Sink: [name] - [description] - Depends on: [resources]
+8. Copy: [name] - [description] - Depends on: [resources]
+9. Generate mock data: [datasource_name]
+10. Append existing fixture: [fixture_pathname] - Target: [datasource_name]
 
 <dev_notes>
 You can skip steps where resources will not be created or updated.
+Always add 'Create secrets' as the FIRST step if any secrets/environment variables are required for the implementation. This step should include ALL required secrets at once.
 Always add 'Generate mock data' step if a landing datasource was created without providing a fixture file.
 Always add 'Append existing fixture' step if a landing datasource was created after providing a fixture file.
 Solve the specific user request, do not add extra steps that are not related to the user request.
@@ -103,8 +101,6 @@ sql_instructions = """
     - ONLY SELECT statements are allowed in any sql query.
     - When using functions try always ClickHouse functions first, then SQL functions.
     - Parameters are never quoted in any case.
-    - Use the following syntax in the SQL section for the iceberg table function: iceberg('s3://bucket/path/to/table', {{tb_secret('aws_access_key_id')}}, {{tb_secret('aws_secret_access_key')}})
-    - Use the following syntax in the SQL section for the postgres table function: postgresql('host:port', 'database', 'table', {{tb_secret('db_username')}}, {{tb_secret('db_password')}}), 'schema')
 </sql_instructions>
 """
 
@@ -158,6 +154,29 @@ def resources_prompt(project: Project) -> str:
     return resources_content + "\n" + fixture_content
 
 
+def secrets_prompt(project: Project) -> str:
+    """Generate a prompt showing available secrets from .env.local file."""
+    secrets = project.get_secrets()
+
+    if not secrets:
+        return "# Environment variables from .env.local:\nNo secrets found in .env.local file"
+
+    secrets_content = "# Environment variables from .env.local:\n"
+    secrets_list = []
+
+    for key, value in sorted(secrets.items()):
+        secret = {
+            "key": key,
+            "value": value,
+        }
+        secrets_list.append(secret)
+
+    if secrets_list:
+        secrets_content = format_as_xml(secrets_list, root_tag="secrets", item_tag="secret")
+
+    return secrets_content
+
+
 def tests_files_prompt(project: Project) -> str:
     files = project.get_project_files()
     test_files = project.get_test_files()
@@ -640,6 +659,72 @@ Do not add additional code explanation summary unless requested by the user. Aft
 IMPORTANT: DO NOT ADD ANY COMMENTS unless asked by the user.
 """
 
+secrets_instructions = """
+# Working with secrets:
+- The syntax to use a secret is `{{ tb_secret("SECRET_NAME", "DEFAULT_VALUE_OPTIONAL") }}`.
+- Secrets are used for sensitive credentials in the following cases:
+  - Connection files
+  - Pipe files in the SQL section (Remember to add `%` on top of the query to make it dynamic)
+  - All credentials needed to access Postgres and Iceberg external tables
+- Do NOT use dynamic parameters instead of secrets, in the cases where secrets are needed.
+- Secrets in pipe files do not allow default values.
+- Secrets in connection files allow default values.
+"""
+
+external_tables_instructions = """
+# Querying external tables:
+When users ask to query a Postgres or Iceberg table, do not create connections, just use the following syntax in the SQL section:
+## Iceberg table example:
+```sql
+FROM iceberg('s3://bucket/path/to/table', {{tb_secret('aws_access_key_id')}}, {{tb_secret('aws_secret_access_key')}})
+```
+## Postgres table example:
+```sql
+FROM postgresql({{ tb_secret("db_host_port") }}, 'database', 'table', {{tb_secret('db_username')}}, {{tb_secret('db_password')}}), 'schema_optional')
+```
+<dev_notes>
+- Do not split the host and port in multiple secrets, use the secret as a whole.
+</dev_notes>
+"""
+
+connection_instructions = """
+# Working with connections files:
+- Content cannot be empty.
+- The connection names must be unique.
+- No indentation is allowed for property names
+- We support kafka, gcs and s3 connections for now
+- If a user asks for a non supported connection type, just say that it is not supported and do not try to create it.
+
+## Kafka connection example:
+```
+TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("PRODUCTION_KAFKA_SERVERS", "localhost:9092") }}
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY {{ tb_secret("PRODUCTION_KAFKA_USERNAME", "") }}
+KAFKA_SECRET {{ tb_secret("PRODUCTION_KAFKA_PASSWORD", "") }}
+```
+
+## S3 connection example:
+```
+TYPE s3
+S3_REGION {{ tb_secret("PRODUCTION_S3_REGION", "") }}
+S3_ARN {{ tb_secret("PRODUCTION_S3_ARN", "") }}
+```
+
+## GCS service account connection example:
+```
+TYPE gcs
+GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON {{ tb_secret("PRODUCTION_GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON", "") }}
+```
+
+## GCS HMAC connection example:
+```
+TYPE gcs
+GCS_HMAC_ACCESS_ID {{ tb_secret("gcs_hmac_access_id") }}
+GCS_HMAC_SECRET {{ tb_secret("gcs_hmac_secret") }}
+```
+"""
 
 agent_system_prompt = f"""
 You are a Tinybird Code, an agentic CLI that can help users to work with Tinybird.
@@ -720,13 +805,11 @@ IMPORTANT: Every time you finish a plan and start a new resource creation or upd
 {sql_agent_instructions}
 {sql_instructions}
 
-# Working with connections files:
-{connection_instructions}
+{secrets_instructions}
 
-# Connection examples:
-Kafka: {kafka_connection_example}
-S3: {s3_connection_example}
-GCS: {gcs_connection_example}
+{external_tables_instructions}
+
+{connection_instructions}
 
 {explore_data_instructions}
 

tinybird-0.0.1.dev270/tinybird/tb/modules/agent/tools/secret.py

@@ -0,0 +1,113 @@
+from pathlib import Path
+
+import click
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import (
+    AgentRunCancelled,
+    TinybirdAgentContext,
+    create_terminal_box,
+    show_confirmation,
+    show_input,
+)
+from tinybird.tb.modules.exceptions import CLIBuildException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def create_or_update_secrets(ctx: RunContext[TinybirdAgentContext], secrets: dict[str, str]) -> str:
+    """Creates or updates multiple secrets in the .env.local file in the project folder.
+
+    This function will:
+    1. Check if .env.local exists in the project folder, create it if it doesn't
+    2. Create or update all specified secret keys with their given values
+    3. Handle existing keys by updating their values
+    4. Show a single confirmation for all secrets
+
+    Args:
+        secrets (dict[str, str]): Dictionary of environment variable keys and values. Required.
+
+    Returns:
+        str: Confirmation message about the secrets creation/update.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+
+        env_file_path = Path(ctx.deps.folder) / ".env.local"
+
+        # Read existing content if file exists
+        existing_content = ""
+        if env_file_path.exists():
+            existing_content = env_file_path.read_text()
+
+        # Parse existing environment variables
+        env_vars = {}
+        for line in existing_content.splitlines():
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                k, v = line.split("=", 1)
+                env_vars[k.strip()] = v.strip()
+
+        # Update or add the new secrets
+        new_keys = []
+        updated_keys = []
+        for key, value in secrets.items():
+            if key in env_vars:
+                updated_keys.append(key)
+            else:
+                new_keys.append(key)
+            env_vars[key] = value
+
+        # Generate new content
+        new_content = "\n".join([f"{k}={v}" for k, v in sorted(env_vars.items())])
+        if new_content:
+            new_content += "\n"
+
+        # Show preview
+        action = "Update" if env_file_path.exists() and updated_keys else "Create"
+        if new_keys and updated_keys:
+            action = "Create/Update"
+
+        preview_content = create_terminal_box(new_content, title=".env.local")
+        click.echo(preview_content)
+
+        confirmation = show_confirmation(
+            title=f"{action} {len(secrets)} secret(s) in .env.local?",
+            skip_confirmation=ctx.deps.dangerously_skip_permissions,
+        )
+
+        if confirmation == "review":
+            feedback = show_input(ctx.deps.workspace_name)
+            ctx.deps.thinking_animation.start()
+            return f"User did not confirm the secret changes and gave the following feedback: {feedback}"
+
+        # Write the file
+        action = "Updating" if env_file_path.exists() and updated_keys else "Creating"
+        click.echo(FeedbackManager.highlight(message=f"» {action} secrets in .env.local..."))
+        env_file_path.write_text(new_content)
+        ctx.deps.load_secrets_local()
+        ctx.deps.build_project(test=False, silent=True, load_fixtures=False)
+
+        # Generate success message
+        result_parts = []
+        if new_keys:
+            result_parts.append(f"created {len(new_keys)} new secret(s): {', '.join(new_keys)}")
+        if updated_keys:
+            result_parts.append(f"updated {len(updated_keys)} existing secret(s): {', '.join(updated_keys)}")
+
+        result_msg = " and ".join(result_parts)
+        ctx.deps.thinking_animation.start()
+
+        return f"Successfully {result_msg} in .env.local. Project built successfully."
+
+    except AgentRunCancelled as e:
+        raise e
+    except CLIBuildException as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=e))
+        ctx.deps.thinking_animation.start()
+        return f"Error building project: {e}. If the error is related to another resource, fix it and try again."
+    except Exception as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=f"Error managing secrets: {e}"))
+        ctx.deps.thinking_animation.start()
+        return f"Error creating/updating secrets: {e}"

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird/tb/modules/agent/utils.py

@@ -61,6 +61,7 @@ class TinybirdAgentContext(BaseModel):
     get_connection_datafile_cloud: Callable[..., str]
     get_connection_datafile_local: Callable[..., str]
     run_tests: Callable[..., Optional[str]]
+    load_secrets_local: Callable[..., None]
     dangerously_skip_permissions: bool
     token: str
     user_token: str

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird/tb/modules/project.py

@@ -82,6 +82,27 @@ class Project:
             test_files.append(test_file)
         return test_files
 
+    def get_secrets(self) -> dict[str, str]:
+        """Load secrets from .env.local file in the project folder."""
+        secrets: dict[str, str] = {}
+        env_file_path = Path(self.folder) / ".env.local"
+
+        if not env_file_path.exists():
+            return secrets
+
+        try:
+            content = env_file_path.read_text()
+            for line in content.splitlines():
+                line = line.strip()
+                if line and not line.startswith("#") and "=" in line:
+                    key, value = line.split("=", 1)
+                    secrets[key.strip()] = value.strip()
+        except Exception:
+            # If there's any error reading the file, return empty dict
+            pass
+
+        return secrets
+
     def get_resource_path(self, resource_name: str, resource_type: str) -> str:
         full_path = next(
             (p for p in self.get_project_files() if p.endswith("/" + resource_name + f".{resource_type}")), ""

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev269
+Version: 0.0.1.dev270
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev269 → tinybird-0.0.1.dev270}/tinybird.egg-info/SOURCES.txt

@@ -106,6 +106,7 @@ tinybird/tb/modules/agent/tools/mock.py
 tinybird/tb/modules/agent/tools/plan.py
 tinybird/tb/modules/agent/tools/request_endpoint.py
 tinybird/tb/modules/agent/tools/run_command.py
+tinybird/tb/modules/agent/tools/secret.py
 tinybird/tb/modules/agent/tools/test.py
 tinybird/tb/modules/datafile/build.py
 tinybird/tb/modules/datafile/build_common.py