tinybird 0.0.1.dev252__py3-none-any.whl → 0.0.1.dev254__py3-none-any.whl

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev252'
-__revision__ = 'd70cad6'
+__version__ = '0.0.1.dev254'
+__revision__ = '269e345'

tinybird/tb/modules/agent/agent.py

@@ -30,22 +30,24 @@ from tinybird.tb.modules.agent.memory import clear_history
 from tinybird.tb.modules.agent.models import create_model, model_costs
 from tinybird.tb.modules.agent.prompts import (
     datafile_instructions,
+    endpoint_optimization_instructions,
     plan_instructions,
     resources_prompt,
     sql_instructions,
 )
-from tinybird.tb.modules.agent.tools.append import append
+from tinybird.tb.modules.agent.tools.analyze import analyze_file, analyze_url
+from tinybird.tb.modules.agent.tools.append import append_file, append_url
 from tinybird.tb.modules.agent.tools.build import build
 from tinybird.tb.modules.agent.tools.create_datafile import create_datafile
 from tinybird.tb.modules.agent.tools.deploy import deploy
 from tinybird.tb.modules.agent.tools.deploy_check import deploy_check
-from tinybird.tb.modules.agent.tools.explore import explore_data
+from tinybird.tb.modules.agent.tools.execute_query import execute_query
 from tinybird.tb.modules.agent.tools.get_endpoint_stats import get_endpoint_stats
 from tinybird.tb.modules.agent.tools.get_openapi_definition import get_openapi_definition
 from tinybird.tb.modules.agent.tools.mock import mock
 from tinybird.tb.modules.agent.tools.plan import plan
 from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
-from tinybird.tb.modules.agent.tools.read_fixture_data import read_fixture_data
+from tinybird.tb.modules.agent.tools.request_endpoint import request_endpoint
 from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_input
 from tinybird.tb.modules.build_common import process as build_process
 from tinybird.tb.modules.common import _analyze, _get_tb_client
@@ -104,18 +106,21 @@ IMPORTANT: DO NOT ADD ANY COMMENTS unless asked by the user.
 
 # Tools
 You have access to the following tools:
-1. `explore_data` - Explore data in the current workspace
-2. `preview_datafile` - Preview the content of a datafile (datasource, endpoint, materialized, sink, copy, connection).
-3. `create_datafile` - Create a file in the project folder. Confirmation will be asked by the tool before creating the file.
-4. `plan` - Plan the creation or update of resources.
-5. `build` - Build the project.
-6. `deploy` - Deploy the project to Tinybird Cloud.
-7. `deploy_check` - Check if the project can be deployed to Tinybird Cloud before deploying it.
-8. `mock` - Create mock data for a landing datasource.
-9. `read_fixture_data` - Read a fixture data file present in the project folder.
-10. `append` - Append existing fixture to a datasource.
+1. `preview_datafile` - Preview the content of a datafile (datasource, endpoint, materialized, sink, copy, connection).
+2. `create_datafile` - Create a file in the project folder. Confirmation will be asked by the tool before creating the file.
+3. `plan` - Plan the creation or update of resources.
+4. `build` - Build the project.
+5. `deploy` - Deploy the project to Tinybird Cloud.
+6. `deploy_check` - Check if the project can be deployed to Tinybird Cloud before deploying it.
+7. `mock` - Create mock data for a landing datasource.
+8. `analyze_file` - Analyze the content of a fixture file present in the project folder.
+9. `analyze_url` - Analyze the content of an external url.
+9. `append_file` - Append a file present in the project to a datasource.
+10. `append_url` - Append an external url to a datasource.
 11. `get_endpoint_stats` - Get metrics of the requests to an endpoint.
 12. `get_openapi_definition` - Get the OpenAPI definition for all endpoints that are built/deployed to Tinybird Cloud or Local.
+13. `execute_query` - Execute a query against Tinybird Cloud or Local.
+13. `request_endpoint` - Request an endpoint against Tinybird Cloud or Local.
 
 # When creating or updating datafiles:
 1. Use `plan` tool to plan the creation or update of resources.
@@ -131,23 +136,20 @@ You have access to the following tools:
 - If the user does not specify anything about the desired schema, create a schema like this:
 SCHEMA >
     `data` String `json:$`
+
 - Use always json paths with .ndjson files.
 
 # When user wants to optimize an endpoint:
-First check if the query is optimized. E.g is filtering by a column present in the sorting key.
-Avoid when possible to not update the landing datasource.
-Check endpoint stats to analyze how the endpoint is performing. Use `get_endpoint_stats` tool to get the stats.
-When your data is in Tinybird, you can create intermediate data sources to preprocess data and make the endpoints faster. This can be done by using materialized views or copy pipes.
-- Copy pipes capture the result of a pipe at a specific point in time and write it to a target data source. They can run on a schedule or run on demand, making them ideal for event-sourced snapshots, data experimentation, and deduplication with snapshots.
-- Materialized views continuously re-evaluate a query as new events are inserted, maintaining an always up-to-date derived dataset. Unlike copy pipes which create point-in-time snapshots, materialized views provide real-time transformations of your data.
-Each approach has its own strengths and use cases:
-- Use copy pipes when you need scheduled or on-demand snapshots of your data.
-- Use materialized views when you need continuous, real-time transformations.
-Finally, update the existing endpoint itself, do not add a new one.
+{endpoint_optimization_instructions}
 
 IMPORTANT: If the user cancels some of the steps or there is an error in file creation, DO NOT continue with the plan. Stop the process and wait for the user before using any other tool.
 IMPORTANT: Every time you finish a plan and start a new resource creation or update process, create a new plan before starting with the changes.
 
+# Using deployment tools:
+- Use `deploy_check` tool to check if the project can be deployed to Tinybird Cloud before deploying it.
+- Use `deploy` tool to deploy the project to Tinybird Cloud.
+- Only use deployment tools if user explicitly asks for it.
+
 # When planning the creation or update of resources:
 {plan_instructions}
 {datafile_instructions}
@@ -184,7 +186,6 @@ GCS: {gcs_connection_example}
 Today is {datetime.now().strftime("%Y-%m-%d")}
 """,
             tools=[
-                Tool(explore_data, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(preview_datafile, docstring_format="google", require_parameter_descriptions=True, takes_ctx=False),
                 Tool(create_datafile, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(plan, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
@@ -192,8 +193,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 Tool(deploy, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(deploy_check, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(mock, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
-                Tool(read_fixture_data, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
-                Tool(append, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(analyze_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(analyze_url, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(append_file, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(append_url, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(
                     get_endpoint_stats, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True
                 ),
@@ -203,6 +206,8 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                     require_parameter_descriptions=True,
                     takes_ctx=True,
                 ),
+                Tool(execute_query, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(request_endpoint, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
             ],
         )
 
@@ -230,6 +235,8 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 analyze_fixture=partial(analyze_fixture, config=config),
                 execute_cloud_query=partial(execute_cloud_query, config=config),
                 execute_local_query=partial(execute_local_query, config=config),
+                request_endpoint_cloud=partial(request_endpoint_cloud, config=config),
+                request_endpoint_local=partial(request_endpoint_local, config=config),
                 get_project_files=project.get_project_files,
                 folder=folder,
                 thinking_animation=thinking_animation,
@@ -263,12 +270,14 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
             click.echo("\n")
 
 
-def run_agent(config: dict[str, Any], project: Project, dangerously_skip_permissions: bool):
+def run_agent(
+    config: dict[str, Any], project: Project, dangerously_skip_permissions: bool, prompt: Optional[str] = None
+):
     token = config.get("token", None)
     host = config.get("host", None)
     user_token = config.get("user_token", None)
-    workspace_id = config.get("id", None)
-    workspace_name = config.get("name", None)
+    workspace_id = config.get("id", "")
+    workspace_name = config.get("name", "")
     try:
         if not token or not host or not workspace_id or not user_token:
             yes = click.confirm(
@@ -294,8 +303,17 @@ def run_agent(config: dict[str, Any], project: Project, dangerously_skip_permiss
                 )
                 return
 
+                # In print mode, always skip permissions to avoid interactive prompts
+        skip_permissions = dangerously_skip_permissions or (prompt is not None)
+        agent = TinybirdAgent(token, user_token, host, workspace_id, project, skip_permissions)
+
+        # Print mode: run once with the provided prompt and exit
+        if prompt:
+            agent.run(prompt, config, project)
+            return
+
+        # Interactive mode: show banner and enter interactive loop
         display_banner()
-        agent = TinybirdAgent(token, user_token, host, workspace_id, project, dangerously_skip_permissions)
         click.echo()
         click.echo(FeedbackManager.info(message="Describe what you want to create and I'll help you build it"))
         click.echo(FeedbackManager.info(message="Run /help for more commands"))
@@ -420,17 +438,31 @@ def mock_data(
     )
 
 
-def analyze_fixture(config: dict[str, Any], fixture_path: str) -> dict[str, Any]:
+def analyze_fixture(config: dict[str, Any], fixture_path: str, format: str = "json") -> dict[str, Any]:
     local_client = get_tinybird_local_client(config, test=False, silent=True)
-    meta, _data = _analyze(fixture_path, local_client, Path(fixture_path).suffix.lstrip("."))
+    meta, _data = _analyze(fixture_path, local_client, format)
     return meta
 
 
-def execute_cloud_query(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> str:
+def execute_cloud_query(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> dict[str, Any]:
     client = _get_tb_client(config["token"], config["host"])
     return client.query(sql=query, pipeline=pipe_name)
 
 
-def execute_local_query(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> str:
+def execute_local_query(config: dict[str, Any], query: str, pipe_name: Optional[str] = None) -> dict[str, Any]:
     local_client = get_tinybird_local_client(config, test=False, silent=True)
     return local_client.query(sql=query, pipeline=pipe_name)
+
+
+def request_endpoint_cloud(
+    config: dict[str, Any], endpoint_name: str, params: Optional[dict[str, str]] = None
+) -> dict[str, Any]:
+    client = _get_tb_client(config["token"], config["host"])
+    return client.pipe_data(endpoint_name, format="json", params=params)
+
+
+def request_endpoint_local(
+    config: dict[str, Any], endpoint_name: str, params: Optional[dict[str, str]] = None
+) -> dict[str, Any]:
+    local_client = get_tinybird_local_client(config, test=False, silent=True)
+    return local_client.pipe_data(endpoint_name, format="json", params=params)

tinybird/tb/modules/agent/prompts.py

@@ -147,3 +147,240 @@ def get_resource_type(path: Path) -> str:
     elif path.suffix.lower() == ".connection":
         return "connection"
     return "unknown"
+
+
+endpoint_optimization_instructions = """
+<endpoint_optimization_instructions>
+## Endpoint Optimization Instructions
+### Step 1: Identify Performance Issues
+1. Analyze the endpoint's query performance metrics
+2. Look for endpoints with high latency or excessive data scanning
+3. Check read_bytes/write_bytes ratios to detect inefficient operations
+
+### Step 2: Apply the 5-Question Diagnostic Framework
+
+#### Question 1: Are you aggregating or transforming data at query time?
+**Detection:**
+- Look for `count()`, `sum()`, `avg()`, or data type casting in published API endpoints
+- Check if the same calculations are performed on every request
+
+**Fix:**
+- Create Materialized Views to pre-aggregate data at ingestion time
+- Move transformations from query time to ingestion time
+- Example transformation:
+  ```sql
+  -- Before (in endpoint)
+  SELECT date, count(*) as daily_count 
+  FROM events 
+  GROUP BY date
+  
+  -- After (in Materialized View)
+  ENGINE "AggregatingMergeTree"
+  ENGINE_PARTITION_KEY "toYYYYMM(date)"
+  ENGINE_SORTING_KEY "date"
+  AS SELECT 
+    date,
+    count(*) as daily_count
+  FROM events
+  GROUP BY date
+  ```
+
+#### Question 2: Are you filtering by fields in the sorting key?
+**Detection:**
+- Examine WHERE clauses in queries
+- Check if filtered columns are part of the sorting key
+- Look for filters on partition keys instead of sorting keys
+
+**Fix:**
+- Ensure sorting key includes frequently filtered columns
+- Order sorting key columns by selectivity (most selective first)
+- Guidelines:
+  - Use 3-5 columns in sorting key
+  - Place `customer_id` or tenant identifiers first for multi-tenant apps
+  - Avoid `timestamp` as the first sorting key element
+  - Never use partition key for filtering
+
+**Example Fix:**
+```sql
+-- Before
+ENGINE_SORTING_KEY "timestamp, customer_id"
+
+-- After (better for multi-tenant filtering)
+ENGINE_SORTING_KEY "customer_id, timestamp"
+```
+
+#### Question 3: Are you using the best data types?
+**Detection:**
+- Scan for overly large data types:
+  - String where UUID would work
+  - Int64 where UInt32 would suffice
+  - DateTime with unnecessary precision
+  - Nullable columns that could have defaults
+
+**Fix:**
+- Downsize data types:
+  ```sql
+  -- Before
+  id String,
+  count Int64,
+  created_at DateTime64(3),
+  status Nullable(String)
+  
+  -- After
+  id UUID,
+  count UInt32,
+  created_at DateTime,
+  status LowCardinality(String) DEFAULT 'pending'
+  ```
+- Use `LowCardinality()` for strings with <100k unique values
+- Replace Nullable with default values using `coalesce()`
+
+#### Question 4: Are you doing complex operations early in the pipeline?
+**Detection:**
+- Look for JOINs or aggregations before filters
+- Check operation order in multi-node pipes
+
+**Fix:**
+- Reorder operations: Filter → Simple transforms → Complex operations
+- Example:
+  ```sql
+  -- Before
+  SELECT * FROM (
+    SELECT a.*, b.name 
+    FROM events a 
+    JOIN users b ON a.user_id = b.id
+  ) WHERE date >= today() - 7
+  
+  -- After
+  SELECT a.*, b.name 
+  FROM (
+    SELECT * FROM events 
+    WHERE date >= today() - 7
+  ) a
+  JOIN users b ON a.user_id = b.id
+  ```
+
+#### Question 5: Are you joining two or more data sources?
+**Detection:**
+- Identify JOINs in queries
+- Check read_bytes/write_bytes ratio in Materialized Views
+- Look for full table scans on joined tables
+
+**Fix Options:**
+1. Replace JOIN with subquery:
+   ```sql
+   -- Before
+   SELECT e.*, u.name 
+   FROM events e 
+   JOIN users u ON e.user_id = u.id
+   
+   -- After
+   SELECT e.*, 
+     (SELECT name FROM users WHERE id = e.user_id) as name
+   FROM events e
+   WHERE user_id IN (SELECT id FROM users)
+   ```
+
+2. Optimize Materialized View JOINs:
+   ```sql
+   -- Before (inefficient)
+   SELECT a.id, a.value, b.value 
+   FROM a 
+   LEFT JOIN b USING id
+   
+   -- After (optimized)
+   SELECT a.id, a.value, b.value 
+   FROM a 
+   LEFT JOIN (
+     SELECT id, value 
+     FROM b 
+     WHERE b.id IN (SELECT id FROM a)
+   ) b USING id
+   ```
+
+### Step 3: Implementation Actions
+
+#### For Schema Changes:
+1. Update the datasource schema
+2. Update the sorting keys and data types
+3. Update dependent pipes and endpoints
+
+#### For Query Optimizations:
+1. Create Materialized Views for repeated aggregations
+2. Rewrite queries following best practices
+3. Test performance improvements
+
+#### For JOIN Optimizations:
+1. Evaluate if JOIN is necessary
+2. Consider denormalization strategies
+3. Use Copy Pipes for historical data recalculation
+4. Implement filtered JOINs in Materialized Views
+
+#### In general:
+1. If you need to iterate an existing resource, do not create a new iteration, just update it with the needed changes.
+
+## Monitoring and Validation
+
+### Monitoring:
+1. Set up alerts for endpoints exceeding latency thresholds
+2. Review of tinybird.pipe_stats_rt (realtime stats of last 24h) and tinybird.pipe_stats (historical stats aggregated by day)
+3. Track processed data patterns over time
+4. Monitor for query pattern changes
+
+### Success Metrics:
+- Reduced query latency
+- Lower data scanning (read_bytes)
+- Improved read_bytes/write_bytes ratio
+- Consistent sub-second API response times
+
+## Code Templates
+
+### Materialized View Template:
+```sql
+NODE materialized_view_name
+SQL >
+  SELECT 
+    -- Pre-aggregated fields
+    toDate(timestamp) as date,
+    customer_id,
+    count(*) as event_count,
+    sum(amount) as total_amount
+  FROM source_table
+  GROUP BY date, customer_id
+
+TYPE materialized
+DATASOURCE mv_datasource_name
+ENGINE "AggregatingMergeTree"
+ENGINE_PARTITION_KEY "toYYYYMM(date)"
+ENGINE_SORTING_KEY "customer_id, date"
+```
+
+### Optimized Query Template:
+```sql
+NODE endpoint_query
+SQL >
+  -- Step 1: Filter early
+  WITH filtered_data AS (
+    SELECT * FROM events
+    WHERE customer_id = {{ String(customer_id) }}
+      AND date >= {{ Date(start_date) }}
+      AND date <= {{ Date(end_date) }}
+  )
+  -- Step 2: Simple operations
+  SELECT 
+    date,
+    sum(amount) as daily_total
+  FROM filtered_data
+  GROUP BY date
+  ORDER BY date DESC
+```
+
+## Best Practices Summary
+
+1. **Think ingestion-time, not query-time** - Move computations upstream
+2. **Index smartly** - Sorting keys should match filter patterns
+3. **Size appropriately** - Use the smallest viable data types
+4. **Filter first** - Reduce data before complex operations
+5. **JOIN carefully** - Consider alternatives and optimize when necessary
+</endpoint_optimization_instructions>
+"""

tinybird/tb/modules/agent/tools/analyze.py

@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+from urllib.parse import urlparse
+
+import click
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def analyze_file(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
+    """Analyze a fixture data file present in the project folder
+
+    Args:
+        fixture_pathname (str): a path or an external url to a fixture file. Required.
+
+    Returns:
+        str: The content of the fixture data file.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message=f"» Analyzing {fixture_pathname}..."))
+        fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
+
+        if not fixture_path.exists():
+            click.echo(FeedbackManager.error(message=f"No fixture data found for {fixture_pathname}."))
+            ctx.deps.thinking_animation.start()
+            return f"No fixture data found for {fixture_pathname}. Please check the path of the fixture and try again."
+
+        fixture_extension = fixture_path.suffix.lstrip(".")
+        response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path), format=fixture_extension)
+        click.echo(FeedbackManager.success(message="✓ Done!\n"))
+        ctx.deps.thinking_animation.start()
+        # limit content to first 10 rows
+        data = response["preview"]["data"][:10]
+        columns = response["analysis"]["columns"]
+
+        return f"#Result of analysis of {fixture_pathname}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"
+    except Exception as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=f"Error analyzing {fixture_pathname}: {e}"))
+        ctx.deps.thinking_animation.start()
+        return f"Error analyzing {fixture_pathname}: {e}"
+
+
+def analyze_url(ctx: RunContext[TinybirdAgentContext], fixture_url: str):
+    """Analyze a fixture file present in an external url
+
+    Args:
+        fixture_url (str): an external url to a fixture file. Required.
+
+    Returns:
+        str: The analysis with the columns and the first 10 rows of the fixture data file.
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        is_url = urlparse(fixture_url).scheme in ("http", "https")
+        click.echo(FeedbackManager.highlight(message=f"» Analyzing {fixture_url}..."))
+        if not is_url:
+            click.echo(FeedbackManager.error(message=f"{fixture_url} is not a valid url."))
+            ctx.deps.thinking_animation.start()
+            return f"{fixture_url} is not a valid url. Please check the url and try again."
+
+        fixture_extension = fixture_url.split(".")[-1]
+
+        response = ctx.deps.analyze_fixture(fixture_path=fixture_url, format=fixture_extension)
+        click.echo(FeedbackManager.success(message="✓ Done!\n"))
+        ctx.deps.thinking_animation.start()
+        # limit content to first 10 rows
+        data = response["preview"]["data"][:10]
+        columns = response["analysis"]["columns"]
+
+        return f"#Result of analysis of URL {fixture_url}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"
+    except Exception as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=f"Error analyzing {fixture_url}: {e}"))
+        ctx.deps.thinking_animation.start()
+        return f"Error analyzing {fixture_url}: {e}"

tinybird/tb/modules/agent/tools/append.py

@@ -5,7 +5,7 @@ from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_confirmat
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_pathname: str) -> str:
+def append_file(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_pathname: str) -> str:
     """Append existing fixture to a datasource
 
     Args:
@@ -18,7 +18,7 @@ def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_
     try:
         ctx.deps.thinking_animation.stop()
         confirmation = show_confirmation(
-            title=f"Append existing fixture for datasource {datasource_name}?",
+            title=f"Append fixture {fixture_pathname} to datasource {datasource_name}?",
             skip_confirmation=ctx.deps.dangerously_skip_permissions,
         )
 
@@ -45,3 +45,43 @@ def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_
         click.echo(FeedbackManager.error(message=e))
         ctx.deps.thinking_animation.start()
         return f"Error appending fixture {fixture_pathname} to {datasource_name}: {e}"
+
+
+def append_url(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_url: str) -> str:
+    """Append existing fixture to a datasource
+
+    Args:
+        datasource_name: Name of the datasource to append fixture to
+        fixture_url: external url to the fixture file to append
+
+    Returns:
+        str: Message indicating the success or failure of the appending
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        confirmation = show_confirmation(
+            title=f"Append URL {fixture_url} to datasource {datasource_name}?",
+            skip_confirmation=ctx.deps.dangerously_skip_permissions,
+        )
+
+        if confirmation == "review":
+            click.echo()
+            feedback = show_input(ctx.deps.workspace_name)
+            ctx.deps.thinking_animation.start()
+            return f"User did not confirm appending URL {fixture_url} and gave the following feedback: {feedback}"
+
+        if confirmation == "cancel":
+            ctx.deps.thinking_animation.start()
+            return f"User rejected appending URL {fixture_url}. Skip this step"
+
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending {fixture_url} to {datasource_name}..."))
+        ctx.deps.append_data(datasource_name=datasource_name, path=fixture_url)
+        click.echo(FeedbackManager.success(message=f"✓ Data appended to {datasource_name}"))
+        ctx.deps.thinking_animation.start()
+        return f"Data appended to {datasource_name}"
+    except Exception as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=e))
+        ctx.deps.thinking_animation.start()
+        return f"Error appending URL {fixture_url} to {datasource_name}: {e}"

tinybird/tb/modules/agent/tools/deploy_check.py

@@ -1,7 +1,7 @@
 import click
 from pydantic_ai import RunContext
 
-from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_confirmation, show_input
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
@@ -9,6 +9,21 @@ def deploy_check(ctx: RunContext[TinybirdAgentContext]) -> str:
     """Check that project can be deployed"""
     try:
         ctx.deps.thinking_animation.stop()
+        confirmation = show_confirmation(
+            title="Check that project can be deployed?",
+            skip_confirmation=ctx.deps.dangerously_skip_permissions,
+        )
+
+        if confirmation == "review":
+            click.echo()
+            feedback = show_input(ctx.deps.workspace_name)
+            ctx.deps.thinking_animation.start()
+            return f"User did not confirm deployment check and gave the following feedback: {feedback}"
+
+        if confirmation == "cancel":
+            ctx.deps.thinking_animation.start()
+            return "User cancelled deployment check. Stop deployment check."
+
         ctx.deps.deploy_check_project()
         ctx.deps.thinking_animation.start()
         return "Project can be deployed"

tinybird/tb/modules/agent/tools/execute_query.py

@@ -0,0 +1,61 @@
+import click
+import humanfriendly
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def execute_query(ctx: RunContext[TinybirdAgentContext], query: str, task: str, cloud: bool = True):
+    """Execute a query:
+
+    Args:
+        query (str): The query to execute. Required.
+        task (str): The purpose of the query. Required.
+        cloud (bool): Whether to execute the query on cloud or local. Optional.
+
+    Returns:
+        str: The result of the query.
+    """
+    try:
+        cloud_or_local = "cloud" if cloud else "local"
+        ctx.deps.thinking_animation.stop()
+
+        click.echo(FeedbackManager.highlight(message=f"» Executing query to {cloud_or_local}:\n{query}\n"))
+
+        is_templating = query.strip().startswith("%")
+        query_format = "FORMAT JSON"
+        if is_templating:
+            query = query.strip()
+            query = f"%\nSELECT * FROM ({query}) {query_format}"
+        else:
+            query = f"SELECT * FROM ({query}) {query_format}"
+
+        execute_query = ctx.deps.execute_cloud_query if cloud else ctx.deps.execute_local_query
+        result = execute_query(query=query)
+        stats = result["statistics"]
+        seconds = stats["elapsed"]
+        rows_read = humanfriendly.format_number(stats["rows_read"])
+        bytes_read = humanfriendly.format_size(stats["bytes_read"])
+
+        click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
+
+        if not result["data"]:
+            click.echo(FeedbackManager.info_no_rows())
+        else:
+            echo_safe_humanfriendly_tables_format_smart_table(
+                data=[d.values() for d in result["data"]], column_names=result["data"][0].keys()
+            )
+        ctx.deps.thinking_animation.start()
+        result["data"] = result["data"][:10]
+        return f"Result for task '{task}' in {cloud_or_local} environment: {result}. The user is being shown the full result in the console but this message only contains the first 10 rows."
+    except Exception as e:
+        error = str(e)
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=error))
+        ctx.deps.thinking_animation.start()
+        if "not found" in error.lower() and cloud:
+            return f"Error executing query: {error}. Please run the query against Tinybird local instead of cloud."
+        else:
+            return f"Error executing query: {error}. Please try again."

tinybird/tb/modules/agent/tools/request_endpoint.py

@@ -0,0 +1,65 @@
+from typing import Optional
+
+import click
+import humanfriendly
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def request_endpoint(
+    ctx: RunContext[TinybirdAgentContext],
+    endpoint_name: str,
+    params: Optional[dict[str, str]] = None,
+    cloud: bool = True,
+):
+    """Request an endpoint:
+
+    Args:
+        endpoint_name (str): The name of the endpoint to request. Required.
+        params (dict): The parameters to pass to the endpoint. Optional.
+        cloud (bool): Whether to request the endpoint on cloud or local. Optional.
+
+    Returns:
+        str: The result of the query.
+    """
+    try:
+        cloud_or_local = "cloud" if cloud else "local"
+        ctx.deps.thinking_animation.stop()
+
+        click.echo(
+            FeedbackManager.highlight(
+                message=f"» Calling endpoint {endpoint_name} in {cloud_or_local} environment with params {params}"
+            )
+        )
+
+        request_endpoint = ctx.deps.request_endpoint_cloud if cloud else ctx.deps.request_endpoint_local
+        result = request_endpoint(endpoint_name=endpoint_name, params=params)
+        stats = result["statistics"]
+        seconds = stats["elapsed"]
+        rows_read = humanfriendly.format_number(stats["rows_read"])
+        bytes_read = humanfriendly.format_size(stats["bytes_read"])
+
+        click.echo(FeedbackManager.success_print_pipe(pipe=endpoint_name))
+        click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
+
+        if not result["data"]:
+            click.echo(FeedbackManager.info_no_rows())
+        else:
+            echo_safe_humanfriendly_tables_format_smart_table(
+                data=[d.values() for d in result["data"][:5]], column_names=result["data"][0].keys()
+            )
+            click.echo("Showing first 5 results\n")
+        ctx.deps.thinking_animation.start()
+        return f"Result for endpoint {endpoint_name} with params {params} in {cloud_or_local} environment: {result}. Do not show result is already shown in the console."
+    except Exception as e:
+        error = str(e)
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=error))
+        ctx.deps.thinking_animation.start()
+        if "not found" in error.lower() and cloud:
+            return f"Error executing query: {error}. Please run the query against Tinybird local instead of cloud."
+        else:
+            return f"Error executing query: {error}. Please try again."

tinybird/tb/modules/agent/utils.py

@@ -41,8 +41,10 @@ class TinybirdAgentContext(BaseModel):
     mock_data: Callable[..., list[dict[str, Any]]]
     append_data: Callable[..., None]
     analyze_fixture: Callable[..., dict[str, Any]]
-    execute_cloud_query: Callable[..., str]
-    execute_local_query: Callable[..., str]
+    execute_cloud_query: Callable[..., dict[str, Any]]
+    execute_local_query: Callable[..., dict[str, Any]]
+    request_endpoint_cloud: Callable[..., dict[str, Any]]
+    request_endpoint_local: Callable[..., dict[str, Any]]
     dangerously_skip_permissions: bool
     token: str
     user_token: str

tinybird/tb/modules/cli.py

@@ -86,6 +86,10 @@ agent_mode_flag = os.environ.get("TB_AGENT_MODE", "false") == "true"
     help="Skip permissions check in agent mode.",
     hidden=True,
 )
+@click.option(
+    "--p",
+    help="Run agent in print mode with the provided prompt and exit.",
+)
 @click.version_option(version=VERSION)
 @click.pass_context
 def cli(
@@ -101,6 +105,7 @@ def cli(
     output: str,
     max_depth: int,
     dangerously_skip_permissions: bool,
+    p: Optional[str],
 ) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
@@ -202,9 +207,10 @@ def cli(
     ctx.ensure_object(dict)["output"] = output
 
     is_agent_mode = agent_mode_flag and ctx.invoked_subcommand is None
+    is_print_mode = p is not None
 
-    if is_agent_mode:
-        run_agent(config, project, dangerously_skip_permissions)
+    if is_agent_mode or is_print_mode:
+        run_agent(config, project, dangerously_skip_permissions, prompt=p)
 
 
 @cli.command(hidden=True)

{tinybird-0.0.1.dev252.dist-info → tinybird-0.0.1.dev254.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev252
+Version: 0.0.1.dev254
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev252.dist-info → tinybird-0.0.1.dev254.dist-info}/RECORD

@@ -17,7 +17,7 @@ tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1w
 tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
 tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
 tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
-tinybird/tb/__cli__.py,sha256=tux3rAKuHbXOtIaG9CZQxDsezngoOjC_cZmI7OXF1UY,247
+tinybird/tb/__cli__.py,sha256=dY306yA2ppVsNM4k4b-zkQx3D8E5suyaU1zBkami7R0,247
 tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
 tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
 tinybird/tb/client.py,sha256=pJbdkWMXGAqKseNAvdsRRnl_c7I-DCMB0dWCQnG82nU,54146
@@ -25,7 +25,7 @@ tinybird/tb/config.py,sha256=mhMTGnMB5KcxGoh3dewIr2Jjsa6pHE183gCPAQWyp6o,3973
 tinybird/tb/modules/build.py,sha256=efD-vamK1NPaDo9R86Hn8be2DYoW0Hh5bZiH7knK5dk,7790
 tinybird/tb/modules/build_common.py,sha256=rWhemU8bk0ZE2eiwZDaTmV9cPabDGGlyc2WnRxfhT0M,12859
 tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
-tinybird/tb/modules/cli.py,sha256=DnNCC1HBe42CNCYIa8FR7YfbUNo9Mbd4ATTVShK8f6Q,16573
+tinybird/tb/modules/cli.py,sha256=1kErLFhxgMWldbE7P4-bkPUcug8ymRyXDHRG9-vGb_4,16755
 tinybird/tb/modules/common.py,sha256=jTTaDDHrZREt--032XhP6GkbfFwC79YJ5aH1Sl7bmbo,81925
 tinybird/tb/modules/config.py,sha256=gK7rgaWTDd4ZKCrNEg_Uemr26EQjqWt6TjyQKujxOws,11462
 tinybird/tb/modules/connection.py,sha256=-MY56NUAai6EMC4-wpi7bT0_nz_SA8QzTmHkV7HB1IQ,17810
@@ -68,26 +68,28 @@ tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,
 tinybird/tb/modules/workspace.py,sha256=Q_8HcxMsNg8QG9aBlwcWS2umrDP5IkTIHqqz3sfmGuc,11341
 tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
 tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
-tinybird/tb/modules/agent/agent.py,sha256=Uxo2FNNu-YW-J-AbZizZe0eKYKdZRwbPJCgA9EsXlgs,21208
+tinybird/tb/modules/agent/agent.py,sha256=lemqCQyYYnoSqp3gVPMAc0J7b4OrLy8sCpUk1_Vy9bo,22150
 tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
 tinybird/tb/modules/agent/banner.py,sha256=KX_e467uiy1gWOZ4ofTZt0GCFGQqHQ_8Ob27XLQqda0,3053
 tinybird/tb/modules/agent/memory.py,sha256=H6SJK--2L5C87B7AJd_jMqsq3sCvFvZwZXmajuT0GBE,1171
 tinybird/tb/modules/agent/models.py,sha256=LW1D27gjcd_jwFmghEzteCgToDfodX2B6B5S8BYbysw,735
-tinybird/tb/modules/agent/prompts.py,sha256=wbe6vUnm-fskceWgP13R5VW1v_YF7_wLDe-wBN6rlWw,6998
-tinybird/tb/modules/agent/utils.py,sha256=fOliFWZ1A6_SBjwreJDid53cMFjH1Ah4pe5DumPSolE,26111
+tinybird/tb/modules/agent/prompts.py,sha256=-2Gazjrh3NGjSqtw9yZ8Up-tQLDD-kKGj7rwPSk71oY,13377
+tinybird/tb/modules/agent/utils.py,sha256=me5-kflBxV4N4psBSxI3vAfmFvAtsSnboc7ND3M9omw,26249
 tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tinybird/tb/modules/agent/tools/append.py,sha256=cBdKBhUW0kooIbywtZNd_bhxbbMyf5NUKHwanAl5BmA,2042
+tinybird/tb/modules/agent/tools/analyze.py,sha256=7oxJ3waCS24Qja_k5GUB59_XiHTG9pCewZogOXhH0cA,3495
+tinybird/tb/modules/agent/tools/append.py,sha256=brd7OFSFJCC3-k6_1p2Qlcg_v2cDlpL_iBUtWtpiqfw,3821
 tinybird/tb/modules/agent/tools/build.py,sha256=LhzJMx6tbxC7gogIrxhfKJc-SDgoSR-FC6IunfaCdn8,758
 tinybird/tb/modules/agent/tools/create_datafile.py,sha256=wcPcChACTIFKw0lKFTlhm0sWJKhQkMLPLnGNpKyeETA,2962
 tinybird/tb/modules/agent/tools/deploy.py,sha256=WrsSlaufKGOBx0S13uoMQQH2DnKue5LQ231Rx4RXh2I,1443
-tinybird/tb/modules/agent/tools/deploy_check.py,sha256=VqMYC7l3_cihmmM_pi8w1t8rJ3P0xDc7pHs_st9k-9Q,684
+tinybird/tb/modules/agent/tools/deploy_check.py,sha256=LBE8aENYvCEaxbVTVKMSI2NGGiHeh-y60_MPlfrcvFk,1331
+tinybird/tb/modules/agent/tools/execute_query.py,sha256=s6QCIe8iD44_XZWgN-zYvWtXPEgJb2kGx6zgXCJbhKc,2642
 tinybird/tb/modules/agent/tools/explore.py,sha256=ihALc_kBcsjrKT3hZyicqyIowB0g_K3AtNNi-5uz9-8,412
 tinybird/tb/modules/agent/tools/get_endpoint_stats.py,sha256=_3wAvDykJitIOb5BRnP7wCy6y06y1qlULHLWB-MvS2M,1705
 tinybird/tb/modules/agent/tools/get_openapi_definition.py,sha256=9cQ-SUeB1NVhPJN1s8aQh9KQxqI9-DEEW1Ot5r2JbOk,1575
 tinybird/tb/modules/agent/tools/mock.py,sha256=Omog_gdEdm8YuBXNrJdHwxHqjL_ji9UIr75mALF4ozI,3408
 tinybird/tb/modules/agent/tools/plan.py,sha256=pr6LnItz6vlOeCG8GE459ExsrBEG0KLx-g02SZGNjXU,1217
 tinybird/tb/modules/agent/tools/preview_datafile.py,sha256=e9q5fR0afApcrntzFrnuHmd10ex7MG_GM6T0Pwc9bRI,850
-tinybird/tb/modules/agent/tools/read_fixture_data.py,sha256=rvTdVlZsu3rQTSWqXzpFt4LEwnBcMLIT8hlI5C7MVN4,1430
+tinybird/tb/modules/agent/tools/request_endpoint.py,sha256=iwzjYLtX_4YS7b9KzRwPGtVkx2UNujMAXv8m7mm8Fac,2683
 tinybird/tb/modules/datafile/build.py,sha256=NFKBrusFLU0WJNCXePAFWiEDuTaXpwc0lHlOQWEJ43s,51117
 tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
 tinybird/tb/modules/datafile/build_datasource.py,sha256=Ra8pVQBDafbFRUKlhpgohhTsRyp_ADKZJVG8Gd69idY,17227
@@ -108,8 +110,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev252.dist-info/METADATA,sha256=5RXjrDqF3SKasGIPyA3n9a5ZbKyFwrNxwxP-SwnVprk,1733
-tinybird-0.0.1.dev252.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev252.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev252.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev252.dist-info/RECORD,,
+tinybird-0.0.1.dev254.dist-info/METADATA,sha256=OJ8Hn_avSbp70iyOkjiWretCRbTvI-jUk514rgWiA40,1733
+tinybird-0.0.1.dev254.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev254.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev254.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev254.dist-info/RECORD,,

tinybird/tb/modules/agent/tools/read_fixture_data.py

@@ -1,36 +0,0 @@
-import json
-from pathlib import Path
-
-import click
-from pydantic_ai import RunContext
-
-from tinybird.tb.modules.agent.utils import TinybirdAgentContext
-from tinybird.tb.modules.feedback_manager import FeedbackManager
-
-
-def read_fixture_data(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
-    """Read fixture data in the project folder
-
-    Args:
-        fixture_pathname (str): a path to a fixture file. Required.
-
-    Returns:
-        str: The content of the fixture data file.
-    """
-    ctx.deps.thinking_animation.stop()
-    click.echo(FeedbackManager.highlight(message=f"» Analyzing {fixture_pathname}..."))
-    fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
-
-    if not fixture_path.exists():
-        click.echo(FeedbackManager.error(message=f"No fixture data found for {fixture_pathname}."))
-        ctx.deps.thinking_animation.start()
-        return f"No fixture data found for {fixture_pathname}. Please check the name of the fixture and try again."
-
-    response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path))
-    click.echo(FeedbackManager.success(message="✓ Done!\n"))
-    ctx.deps.thinking_animation.start()
-    # limit content to first 10 rows
-    data = response["preview"]["data"][:10]
-    columns = response["analysis"]["columns"]
-
-    return f"#Result of analysis of {fixture_pathname}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"