tinybird 0.0.1.dev246__py3-none-any.whl → 0.0.1.dev248__py3-none-any.whl

tinybird/ch_utils/constants.py

@@ -255,4 +255,6 @@ VALID_QUERY_FORMATS = (
     "JSONStrings",
     "Prometheus",
     "Native",
+    "RowBinaryWithNamesAndTypes",
+    "TabSeparated",
 )

tinybird/prompts.py

@@ -776,6 +776,8 @@ datasource_instructions = """
     - Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
     - Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
     - If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
+    - Unless the user asks for them, do not include ENGINE_PARTITION_KEY and ENGINE_PRIMARY_KEY.
+    - DateTime64 type without precision is not supported. Use DateTime64(3) instead.
 </datasource_file_instructions>
 """
 

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev246'
-__revision__ = '5c11131'
+__version__ = '0.0.1.dev248'
+__revision__ = '8eed30e'

tinybird/tb/modules/agent/agent.py

@@ -2,6 +2,7 @@ import subprocess
 import sys
 from datetime import datetime
 from functools import partial
+from pathlib import Path
 from typing import Any
 
 import click
@@ -28,34 +29,41 @@ from tinybird.tb.client import TinyB
 from tinybird.tb.modules.agent.animations import ThinkingAnimation
 from tinybird.tb.modules.agent.banner import display_banner
 from tinybird.tb.modules.agent.memory import clear_history, load_history
-from tinybird.tb.modules.agent.models import create_model
+from tinybird.tb.modules.agent.models import create_model, model_costs
 from tinybird.tb.modules.agent.prompts import (
     datafile_instructions,
     plan_instructions,
     resources_prompt,
     sql_instructions,
 )
+from tinybird.tb.modules.agent.tools.append import append
 from tinybird.tb.modules.agent.tools.build import build
 from tinybird.tb.modules.agent.tools.create_datafile import create_datafile
 from tinybird.tb.modules.agent.tools.deploy import deploy
 from tinybird.tb.modules.agent.tools.deploy_check import deploy_check
 from tinybird.tb.modules.agent.tools.explore import explore_data
+from tinybird.tb.modules.agent.tools.mock import mock
 from tinybird.tb.modules.agent.tools.plan import plan
 from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
+from tinybird.tb.modules.agent.tools.read_fixture_data import read_fixture_data
 from tinybird.tb.modules.agent.utils import TinybirdAgentContext
 from tinybird.tb.modules.build_common import process as build_process
-from tinybird.tb.modules.common import _get_tb_client
+from tinybird.tb.modules.common import _analyze, _get_tb_client
+from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.deployment_common import create_deployment
-from tinybird.tb.modules.exceptions import CLIBuildException
+from tinybird.tb.modules.exceptions import CLIBuildException, CLIMockException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.login_common import login
+from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
 from tinybird.tb.modules.project import Project
 
 
 class TinybirdAgent:
-    def __init__(self, token: str, host: str, project: Project):
+    def __init__(self, token: str, host: str, project: Project, dangerously_skip_permissions: bool):
         self.token = token
         self.host = host
+        self.dangerously_skip_permissions = dangerously_skip_permissions
         self.project = project
         self.messages: list[ModelMessage] = []
         self.agent = Agent(
@@ -68,7 +76,7 @@ You are an interactive CLI tool that helps users with data engineering tasks. Us
 
 # Tone and style
 You should be concise, direct, and to the point. 
-Remember that your output will be displayed on a command line interface. Your responses can use Github-flavored markdown for formatting.
+Remember that your output will be displayed on a command line interface. Your responses can use Github-flavored markdown for formatting. Do not use emojis.
 Output text to communicate with the user; all text you output outside of tool use is displayed to the user. Only use tools to complete tasks. Never use tools like Bash or code comments as means to communicate with the user during the session.
 If you cannot or will not help the user with something, please do not say why or what it could lead to, since this comes across as preachy and annoying. Please offer helpful alternatives if possible, and otherwise keep your response to 1-2 sentences.
 IMPORTANT: You should minimize output tokens as much as possible while maintaining helpfulness, quality, and accuracy. Only address the specific query or task at hand, avoiding tangential information unless absolutely critical for completing the request. If you can answer in 1-3 sentences or a short paragraph, please do.
@@ -94,6 +102,9 @@ You have access to the following tools:
 5. `build` - Build the project.
 6. `deploy` - Deploy the project to Tinybird Cloud.
 7. `deploy_check` - Check if the project can be deployed to Tinybird Cloud before deploying it.
+8. `mock` - Create mock data for a landing datasource.
+9. `read_fixture_data` - Read a fixture data file present in the project folder.
+10. `append` - Append existing fixture to a datasource.
 
 # When creating or updating datafiles:
 1. Use `plan` tool to plan the creation or update of resources.
@@ -106,6 +117,7 @@ You have access to the following tools:
 8. If the datafile was created successfully, but the built failed, try to fix the error and repeat the process.
 
 IMPORTANT: If the user cancels some of the steps or there is an error in file creation, DO NOT continue with the plan. Stop the process and wait for the user before using any other tool.
+IMPORTANT: Every time you finish a plan and start a new resource creation or update process, create a new plan before starting with the changes.
 
 # When planning the creation or update of resources:
 {plan_instructions}
@@ -150,6 +162,9 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 Tool(build, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(deploy, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
                 Tool(deploy_check, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(mock, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(read_fixture_data, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
+                Tool(append, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
             ],
         )
 
@@ -158,7 +173,7 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
         return self.messages[-5:] if len(self.messages) > 5 else self.messages
 
     def run(self, user_prompt: str, config: dict[str, Any], project: Project) -> None:
-        user_prompt = f"{user_prompt}\n\n# Existing resources in the project:\n{resources_prompt(project)}"
+        user_prompt = f"{user_prompt}\n\n{resources_prompt(project)}"
         client = TinyB(token=self.token, host=self.host)
         folder = self.project.folder
 
@@ -172,34 +187,71 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
                 build_project=partial(build_project, project=project, config=config),
                 deploy_project=partial(deploy_project, project=project, config=config),
                 deploy_check_project=partial(deploy_check_project, project=project, config=config),
+                mock_data=partial(mock_data, project=project, config=config),
+                append_data=partial(append_data, config=config),
+                analyze_fixture=partial(analyze_fixture, config=config),
                 get_project_files=project.get_project_files,
                 folder=folder,
                 thinking_animation=thinking_animation,
                 workspace_name=self.project.workspace_name,
+                dangerously_skip_permissions=self.dangerously_skip_permissions,
             ),
             message_history=self.messages,
         )
         new_messages = result.new_messages()
         self.messages.extend(new_messages)
         thinking_animation.stop()
-
+        usage = result.usage()
+        request_tokens = usage.request_tokens or 0
+        response_tokens = usage.response_tokens or 0
+        total_tokens = usage.total_tokens or 0
+        cost = (
+            request_tokens * model_costs["input_cost_per_token"]
+            + response_tokens * model_costs["output_cost_per_token"]
+        )
+        click.echo()
         click.echo(result.output)
         click.echo("\n")
+        click.echo(f"Input tokens: {request_tokens}")
+        click.echo(f"Output tokens: {response_tokens}")
+        click.echo(f"Total tokens: {total_tokens}")
+        click.echo(f"Cost: ${cost:.6f}")
 
 
-def run_agent(config: dict[str, Any], project: Project):
-    display_banner()
+def run_agent(config: dict[str, Any], project: Project, dangerously_skip_permissions: bool):
+    token = config.get("token", None)
+    host = config.get("host", None)
 
     try:
-        token = config["token"]
-        host = config["host"]
-        agent = TinybirdAgent(token, host, project)
+        if not token or not host:
+            yes = click.confirm(
+                FeedbackManager.warning(
+                    message="Tinybird Code requires authentication. Do you want to authenticate now? [Y/n]"
+                ),
+                prompt_suffix="",
+                show_default=False,
+                default=True,
+            )
+            if yes:
+                click.echo()
+                login(host, auth_host="https://cloud.tinybird.co", workspace=None, interactive=False, method="browser")
+                click.echo()
+                cli_config = CLIConfig.get_project_config()
+                token = cli_config.get_token()
+                host = cli_config.get_host()
+
+            if not token or not host:
+                click.echo(
+                    FeedbackManager.error(message="Tinybird Code requires authentication. Run 'tb login' first.")
+                )
+                return
+
+        display_banner()
+        agent = TinybirdAgent(token, host, project, dangerously_skip_permissions)
         click.echo()
-        if config.get("token"):
-            click.echo(FeedbackManager.info(message="Describe what you want to create and I'll help you build it"))
-            click.echo(FeedbackManager.info(message="Run /help for more commands"))
-        else:
-            click.echo(FeedbackManager.info(message="Run /login to authenticate"))
+        click.echo(FeedbackManager.info(message="Describe what you want to create and I'll help you build it"))
+        click.echo(FeedbackManager.info(message="Run /help for more commands"))
+
         click.echo()
 
     except Exception as e:
@@ -211,7 +263,7 @@ def run_agent(config: dict[str, Any], project: Project):
         while True:
             try:
                 user_input = prompt(
-                    [("class:prompt", "tb » ")],
+                    [("class:prompt", f"tb ({project.workspace_name}) » ")],
                     history=load_history(),
                     cursor=CursorShape.BLOCK,
                     style=Style.from_dict(
@@ -289,3 +341,40 @@ def deploy_check_project(config: dict[str, Any], project: Project) -> None:
         wait=True,
         auto=True,
     )
+
+
+def append_data(config: dict[str, Any], datasource_name: str, path: str) -> None:
+    client = get_tinybird_local_client(config, test=False, silent=False)
+    append_mock_data(client, datasource_name, path)
+
+
+def mock_data(
+    config: dict[str, Any], project: Project, datasource_name: str, data_format: str, rows: int
+) -> list[dict[str, Any]]:
+    client = get_tinybird_local_client(config, test=False, silent=False)
+    cli_config = CLIConfig.get_project_config()
+    datasource_path = project.get_resource_path(datasource_name, "datasource")
+
+    if not datasource_path:
+        raise CLIMockException(f"Datasource {datasource_name} not found")
+
+    datasource_content = Path(datasource_path).read_text()
+    prompt = ""
+    return create_mock_data(
+        datasource_name,
+        datasource_content,
+        rows,
+        prompt,
+        cli_config,
+        config,
+        cli_config.get_user_token() or "",
+        client,
+        data_format,
+        project.folder,
+    )
+
+
+def analyze_fixture(config: dict[str, Any], fixture_path: str) -> dict[str, Any]:
+    local_client = get_tinybird_local_client(config, test=False, silent=True)
+    meta, _data = _analyze(fixture_path, local_client, Path(fixture_path).suffix.lstrip("."))
+    return meta

tinybird/tb/modules/agent/models.py

@@ -18,3 +18,9 @@ def create_model(
         model_name=model,
         provider=AnthropicProvider(anthropic_client=client),
     )
+
+
+model_costs = {
+    "input_cost_per_token": 3e-06,
+    "output_cost_per_token": 1.5e-05,
+}

tinybird/tb/modules/agent/prompts.py

@@ -8,7 +8,7 @@ When asked to create a plan, you MUST respond with this EXACT format and NOTHING
 Plan description: [One sentence describing what will be built]
 
 Steps:
-1. Connection: [name] - [description] - Depends on: none
+1. Connection: [name] - [description]
 2. Datasource: [name] - [description] - Depends on: [connection_name (optional)]
 3. Endpoint: [name] - [description] - Depends on: [resources]
 4. Materialized pipe: [name] - [description] - Depends on: [resources]
@@ -16,9 +16,15 @@ Steps:
 6. Sink: [name] - [description] - Depends on: [resources]
 7. Copy: [name] - [description] - Depends on: [resources]
 8. Build project
+9. Generate mock data: [datasource_name]
+10. Append existing fixture: [fixture_pathname] - Target: [datasource_name]
 
+<dev_notes>
 You can skip steps where resources will not be created or updated.
-Always add BUILD_PROJECT step at the end of the plan.
+Always add 'Build project' step after generating resources.
+Always add 'Generate mock data' step after building project if a landing datasource was created.
+Always add 'Append existing fixture' step after building project if a fixture file was provided at the beginning of the plan.
+</dev_notes>
 
 Resource dependencies:
 [resource_name]: [resources]
@@ -89,31 +95,53 @@ datafile_instructions = """
 
 def resources_prompt(project: Project) -> str:
     files = project.get_project_files()
+    fixture_files = project.get_fixture_files()
 
-    if not files:
-        return "No resources found"
-
-    paths = [Path(file_path) for file_path in files]
-
-    def get_resource_type(path: Path) -> str:
-        if path.suffix.lower() == ".pipe":
-            return Project.get_pipe_type(str(path))
-        elif path.suffix.lower() == ".datasource":
-            return "datasource"
-        elif path.suffix.lower() == ".connection":
-            return "connection"
-        return "unknown"
-
-    return "\n".join(
-        [
-            f"""
-<resource>
-    <path>{file_path.relative_to(project.folder)}</path>
-    <type>{get_resource_type(file_path)}</type>
-    <name>{file_path.stem}</name>
-    <content>{file_path.read_text()}</content>
-</resource>
-"""
-            for file_path in paths
-        ]
-    )
+    resources_content = "# Existing resources in the project:\n"
+    if files:
+        paths = [Path(file_path) for file_path in files]
+
+        resources_content += "\n".join(
+            [
+                f"""
+    <resource>
+        <path>{file_path.relative_to(project.folder)}</path>
+        <type>{get_resource_type(file_path)}</type>
+        <name>{file_path.stem}</name>
+        <content>{file_path.read_text()}</content>
+    </resource>
+    """
+                for file_path in paths
+            ]
+        )
+    else:
+        resources_content += "No resources found"
+
+    fixture_content = "# Fixture files in the project:\n"
+    if fixture_files:
+        paths = [Path(file_path) for file_path in fixture_files]
+        fixture_content += "\n".join(
+            [
+                f"""
+    <fixture>
+        <path>{file_path.relative_to(project.folder)}</path>
+        <name>{file_path.stem}</name>
+    </fixture>
+    """
+                for file_path in paths
+            ]
+        )
+    else:
+        fixture_content += "No fixture files found"
+
+    return resources_content + "\n" + fixture_content
+
+
+def get_resource_type(path: Path) -> str:
+    if path.suffix.lower() == ".pipe":
+        return Project.get_pipe_type(str(path))
+    elif path.suffix.lower() == ".datasource":
+        return "datasource"
+    elif path.suffix.lower() == ".connection":
+        return "connection"
+    return "unknown"

tinybird/tb/modules/agent/tools/append.py

@@ -0,0 +1,55 @@
+import click
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_options
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def get_append_confirmation(datasource_name: str) -> bool:
+    """Get user confirmation for appending existing fixture"""
+    while True:
+        result = show_options(
+            options=["Yes, append existing fixture", "No, and tell Tinybird Code what to do"],
+            title=f"Do you want to append existing fixture for datasource {datasource_name}?",
+        )
+
+        if result is None:  # Cancelled
+            return False
+
+        if result.startswith("Yes"):
+            return True
+        elif result.startswith("No"):
+            return False
+
+        return False
+
+
+def append(ctx: RunContext[TinybirdAgentContext], datasource_name: str, fixture_pathname: str) -> str:
+    """Append existing fixture to a datasource
+
+    Args:
+        datasource_name: Name of the datasource to append fixture to
+        fixture_pathname: Path to the fixture file to append
+
+    Returns:
+        str: Message indicating the success or failure of the appending
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        confirmation = ctx.deps.dangerously_skip_permissions or get_append_confirmation(datasource_name)
+        ctx.deps.thinking_animation.start()
+
+        if not confirmation:
+            return "User rejected appending existing fixture. Skipping..."
+
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending {fixture_pathname} to {datasource_name}..."))
+        ctx.deps.append_data(datasource_name=datasource_name, path=fixture_pathname)
+        click.echo(FeedbackManager.success(message=f"✓ Data appended to {datasource_name}"))
+        ctx.deps.thinking_animation.start()
+        return f"Data appended to {datasource_name}"
+    except Exception as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=e))
+        ctx.deps.thinking_animation.start()
+        return f"Error appending fixture {fixture_pathname} to {datasource_name}: {e}"

tinybird/tb/modules/agent/tools/build.py

@@ -9,6 +9,7 @@ def build(ctx: RunContext[TinybirdAgentContext]) -> str:
     """Build the project"""
     try:
         ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message="\n» Building project..."))
         ctx.deps.build_project(test=False, silent=False)
         ctx.deps.thinking_animation.start()
         return "Project built successfully"

tinybird/tb/modules/agent/tools/create_datafile.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import click
 from pydantic_ai import RunContext
 
-from tinybird.tb.modules.agent.utils import Datafile, TinybirdAgentContext, show_options
+from tinybird.tb.modules.agent.utils import Datafile, TinybirdAgentContext, create_terminal_box, show_options
 from tinybird.tb.modules.exceptions import CLIBuildException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
@@ -39,11 +39,16 @@ def create_datafile(ctx: RunContext[TinybirdAgentContext], resource: Datafile) -
     """
     try:
         ctx.deps.thinking_animation.stop()
-        click.echo(resource.content)
         resource.pathname = resource.pathname.removeprefix("/")
         path = Path(ctx.deps.folder) / resource.pathname
+        content = resource.content
         exists = str(path) in ctx.deps.get_project_files()
-        confirmation = get_resource_confirmation(resource, exists)
+        if exists:
+            content = create_terminal_box(path.read_text(), resource.content, title=resource.pathname)
+        else:
+            content = create_terminal_box(resource.content, title=resource.pathname)
+        click.echo(content)
+        confirmation = ctx.deps.dangerously_skip_permissions or get_resource_confirmation(resource, exists)
 
         if not confirmation:
             ctx.deps.thinking_animation.start()

tinybird/tb/modules/agent/tools/deploy.py

@@ -28,7 +28,7 @@ def deploy(ctx: RunContext[TinybirdAgentContext]) -> str:
     """Deploy the project"""
     try:
         ctx.deps.thinking_animation.stop()
-        confirmation = get_deploy_confirmation()
+        confirmation = ctx.deps.dangerously_skip_permissions or get_deploy_confirmation()
         ctx.deps.thinking_animation.start()
 
         if not confirmation:

tinybird/tb/modules/agent/tools/mock.py

@@ -0,0 +1,59 @@
+import click
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext, show_options
+from tinybird.tb.modules.datafile.fixture import persist_fixture
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+
+
+def get_mock_confirmation(datasource_name: str) -> bool:
+    """Get user confirmation for creating mock data"""
+    while True:
+        result = show_options(
+            options=["Yes, create mock data", "No, and tell Tinybird Code what to do"],
+            title=f"Do you want to generate mock data for datasource {datasource_name}?",
+        )
+
+        if result is None:  # Cancelled
+            return False
+
+        if result.startswith("Yes"):
+            return True
+        elif result.startswith("No"):
+            return False
+
+        return False
+
+
+def mock(ctx: RunContext[TinybirdAgentContext], datasource_name: str, data_format: str, rows: int) -> str:
+    """Create mock data for a datasource
+
+    Args:
+        datasource_name: Name of the datasource to create mock data for
+        data_format: Format of the mock data to create. Options: ndjson, csv
+        rows: Number of rows to create. If not provided, the default is 10
+
+    Returns:
+        str: Message indicating the success or failure of the mock data generation
+    """
+    try:
+        ctx.deps.thinking_animation.stop()
+        confirmation = ctx.deps.dangerously_skip_permissions or get_mock_confirmation(datasource_name)
+        ctx.deps.thinking_animation.start()
+
+        if not confirmation:
+            return "User rejected mock data generation. Skipping..."
+
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.highlight(message=f"\n» Generating mock data for {datasource_name}..."))
+        data = ctx.deps.mock_data(datasource_name=datasource_name, data_format=data_format, rows=rows)
+        fixture_path = persist_fixture(datasource_name, data, ctx.deps.folder, format=data_format)
+        ctx.deps.append_data(datasource_name=datasource_name, path=str(fixture_path))
+        click.echo(FeedbackManager.success(message=f"✓ Data generated for {datasource_name}"))
+        ctx.deps.thinking_animation.start()
+        return f"Mock data generated successfully for datasource {datasource_name}"
+    except Exception as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=e))
+        ctx.deps.thinking_animation.start()
+        return f"Error generating mock data: {e}"

tinybird/tb/modules/agent/tools/plan.py

@@ -35,7 +35,7 @@ def plan(ctx: RunContext[TinybirdAgentContext], plan: str) -> str:
     try:
         ctx.deps.thinking_animation.stop()
         click.echo(plan)
-        confirmation = get_plan_confirmation()
+        confirmation = ctx.deps.dangerously_skip_permissions or get_plan_confirmation()
         ctx.deps.thinking_animation.start()
 
         if not confirmation:

tinybird/tb/modules/agent/tools/read_fixture_data.py

@@ -0,0 +1,28 @@
+import json
+from pathlib import Path
+
+from pydantic_ai import RunContext
+
+from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+
+
+def read_fixture_data(ctx: RunContext[TinybirdAgentContext], fixture_pathname: str):
+    """Read fixture data in the project folder
+
+    Args:
+        fixture_pathname (str): a path to a fixture file. Required.
+
+    Returns:
+        str: The content of the fixture data file.
+    """
+    fixture_path = Path(ctx.deps.folder) / fixture_pathname.lstrip("/")
+
+    if not fixture_path.exists():
+        return f"No fixture data found for {fixture_pathname}. Please check the name of the fixture and try again."
+
+    response = ctx.deps.analyze_fixture(fixture_path=str(fixture_path))
+    # limit content to first 10 rows
+    data = response["preview"]["data"][:10]
+    columns = response["analysis"]["columns"]
+
+    return f"#Result of analysis of {fixture_pathname}:\n##Columns:\n{json.dumps(columns)}\n##Data sample:\n{json.dumps(data)}"