tinybird 0.0.1.dev301__tar.gz → 0.0.1.dev302__tar.gz

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev301
+Version: 0.0.1.dev302
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev301'
-__revision__ = '1ce40fd'
+__version__ = '0.0.1.dev302'
+__revision__ = 'cf4d28e'

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/agent/agent.py

@@ -6,7 +6,6 @@ import sys
 import urllib.parse
 import uuid
 from functools import partial
-from pathlib import Path
 from typing import Any, Callable, Optional
 
 import click
@@ -27,7 +26,6 @@ from tinybird.tb.modules.agent.file_agent import FileAgent
 from tinybird.tb.modules.agent.memory import (
     clear_history,
     clear_messages,
-    get_last_messages_from_last_user_prompt,
     save_messages,
 )
 from tinybird.tb.modules.agent.mock_agent import MockAgent
@@ -72,12 +70,12 @@ from tinybird.tb.modules.common import (
 )
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.deployment_common import create_deployment
-from tinybird.tb.modules.exceptions import CLIBuildException, CLIDeploymentException, CLIMockException
+from tinybird.tb.modules.exceptions import CLIBuildException, CLIDeploymentException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.login_common import login
-from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
+from tinybird.tb.modules.mock_common import append_mock_data
 from tinybird.tb.modules.project import Project
 from tinybird.tb.modules.test_common import run_tests as run_tests_common
 
@@ -92,6 +90,7 @@ class TinybirdAgent:
         project: Project,
         dangerously_skip_permissions: bool,
         prompt_mode: bool,
+        feature: Optional[str] = None,
     ):
         self.token = token
         self.user_token = user_token
@@ -101,14 +100,12 @@ class TinybirdAgent:
         self.project = project
         self.thinking_animation = ThinkingAnimation()
         self.confirmed_plan_id: Optional[str] = None
-        if prompt_mode:
-            self.messages: list[ModelMessage] = get_last_messages_from_last_user_prompt()
-        else:
-            self.messages = []
+        self.feature = feature
+        self.messages: list[ModelMessage] = []
         cli_config = CLIConfig.get_project_config()
         regions = get_regions(cli_config)
         self.agent = Agent(
-            model=create_model(user_token, host, workspace_id),
+            model=create_model(user_token, host, workspace_id, feature=feature),
             deps_type=TinybirdAgentContext,
             instructions=[agent_system_prompt],
             tools=[
@@ -382,7 +379,6 @@ class TinybirdAgent:
             build_project=partial(build_project, project=project, config=config),
             deploy_project=partial(deploy_project, project=project, config=config),
             deploy_check_project=partial(deploy_check_project, project=project, config=config),
-            mock_data=partial(mock_data, project=project, config=config),
             append_data_local=partial(append_data_local, config=config),
             append_data_cloud=partial(append_data_cloud, config=config),
             analyze_fixture=partial(analyze_fixture, config=config),
@@ -511,7 +507,11 @@ class TinybirdAgent:
 
 
 def run_agent(
-    config: dict[str, Any], project: Project, dangerously_skip_permissions: bool, prompt: Optional[str] = None
+    config: dict[str, Any],
+    project: Project,
+    dangerously_skip_permissions: bool,
+    prompt: Optional[str] = None,
+    feature: Optional[str] = None,
 ):
     if not prompt:
         latest_version = CheckPypi().get_latest_version()
@@ -526,7 +526,10 @@ def run_agent(
             )
             if yes:
                 update_cli()
-    click.echo(FeedbackManager.highlight(message="» Initializing Tinybird Code..."))
+
+    if not prompt:
+        click.echo(FeedbackManager.highlight(message="» Initializing Tinybird Code..."))
+
     token = config.get("token", "")
     host = config.get("host", "")
     user_token = config.get("user_token", "")
@@ -585,6 +588,7 @@ def run_agent(
             project,
             dangerously_skip_permissions,
             prompt_mode,
+            feature,
         )
 
         # Print mode: run once with the provided prompt and exit
@@ -763,36 +767,6 @@ def append_data_cloud(config: dict[str, Any], datasource_name: str, path: str) -
     append_mock_data(client, datasource_name, path)
 
 
-def mock_data(
-    config: dict[str, Any],
-    project: Project,
-    datasource_name: str,
-    data_format: str,
-    rows: int,
-    context: Optional[str] = None,
-) -> list[dict[str, Any]]:
-    client = get_tinybird_local_client(config, test=False, silent=False)
-    cli_config = CLIConfig.get_project_config()
-    datasource_path = project.get_resource_path(datasource_name, "datasource")
-
-    if not datasource_path:
-        raise CLIMockException(f"Datasource {datasource_name} not found")
-
-    datasource_content = Path(datasource_path).read_text()
-    return create_mock_data(
-        datasource_name,
-        datasource_content,
-        rows,
-        context or "",
-        cli_config,
-        config,
-        cli_config.get_user_token() or "",
-        client,
-        data_format,
-        project.folder,
-    )
-
-
 def analyze_fixture(config: dict[str, Any], fixture_path: str, format: str = "json") -> dict[str, Any]:
     local_client = get_tinybird_local_client(config, test=False, silent=True)
     meta, _data = _analyze(fixture_path, local_client, format)

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/agent/models.py

@@ -8,7 +8,7 @@ from pydantic_ai.retries import AsyncTenacityTransport, wait_retry_after
 from tenacity import AsyncRetrying, retry_if_exception_type, stop_after_attempt, wait_exponential
 
 
-def create_retrying_client(token: str, workspace_id: str):
+def create_retrying_client(token: str, workspace_id: str, feature: Optional[str] = None):
     """Create a client with smart retry handling for multiple error types."""
 
     def should_retry_status(response):
@@ -29,7 +29,10 @@ def create_retrying_client(token: str, workspace_id: str):
         ),
         validate_response=should_retry_status,
     )
-    return AsyncClient(transport=transport, params={"token": token, "workspace_id": workspace_id})
+    params = {"token": token, "workspace_id": workspace_id}
+    if feature:
+        params["feature"] = feature
+    return AsyncClient(transport=transport, params=params)
 
 
 def create_model(
@@ -38,6 +41,7 @@ def create_model(
     workspace_id: str,
     model: AnthropicModelName = "claude-4-sonnet-20250514",
     run_id: Optional[str] = None,
+    feature: Optional[str] = None,
 ):
     default_headers = {}
     if run_id:

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/agent/utils.py

@@ -44,7 +44,6 @@ class TinybirdAgentContext(BaseModel):
     build_project_test: Callable[..., None]
     deploy_project: Callable[..., None]
     deploy_check_project: Callable[[], None]
-    mock_data: Callable[..., list[dict[str, Any]]]
     append_data_local: Callable[..., None]
     append_data_cloud: Callable[..., None]
     analyze_fixture: Callable[..., dict[str, Any]]

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/cli.py

@@ -488,7 +488,6 @@ def create_ctx_client(
         "logout",
         "update",
         "upgrade",
-        "create",
         "info",
         "tag",
         "push",
@@ -502,7 +501,7 @@ def create_ctx_client(
         return None
 
     commands_always_cloud = ["infra"]
-    commands_always_local = ["build", "dev"]
+    commands_always_local = ["build", "dev", "create"]
     command_always_test = ["test"]
 
     if (

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/create.py

@@ -3,25 +3,22 @@ import os
 import re
 import shutil
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Set
 from urllib.parse import urlparse
 
 import click
 import requests
 
-from tinybird.prompts import claude_rules_prompt, create_prompt, readme_prompt, rules_prompt
-from tinybird.tb.client import TinyB
+from tinybird.prompts import claude_rules_prompt, rules_prompt
+from tinybird.tb.modules.agent import run_agent
 from tinybird.tb.modules.cicd import init_cicd
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import _generate_datafile, generate_datafile
+from tinybird.tb.modules.common import _generate_datafile
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLICreateException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.llm import LLM
-from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
 from tinybird.tb.modules.local_common import get_tinybird_local_client
-from tinybird.tb.modules.mock_common import create_mock_data
 from tinybird.tb.modules.project import Project
 
 
@@ -67,13 +64,9 @@ def create(
         folder_path.mkdir()
 
     try:
-        tb_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
-        user_token: str = ""
         created_something = False
-        if prompt:
-            user_token = ctx_config.get("user_token")
-            if not user_token:
-                raise Exception("This action requires authentication. Run 'tb login' first.")
+        if prompt and not ctx_config.get("user_token"):
+            raise Exception("This action requires authentication. Run 'tb login' first.")
 
         if not validate_project_structure(project):
             click.echo(FeedbackManager.highlight(message="\n» Creating new project structure..."))
@@ -99,28 +92,20 @@ def create(
 
         prompt_result: List[Path] = []
         if prompt:
-            prompt_result = create_resources_from_prompt(tb_client, user_token, prompt, project)
-            result.extend(prompt_result)
-            readme_path = folder_path / "README.md"
-            if readme_path.exists():
-                click.echo(FeedbackManager.highlight(message="\n» Updating project description..."))
-            else:
-                click.echo(FeedbackManager.highlight(message="\n» Creating project description..."))
-                readme_path.touch()
-            llm = LLM(user_token=str(user_token), host=tb_client.host)
-            readme_user_prompt = prompt or ""
-            all_resources_xml = get_resources_xml(project)
-            readme_response = llm.ask(
-                system_prompt=readme_prompt(
-                    readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
-                ),
-                prompt=readme_user_prompt,
-                feature="tb_create_readme",
+            prompt_instructions = (
+                "Create or update the Tinybird datasources, pipes, and connections required to satisfy the following request. "
+                "Do not generate mock data or append data; those steps will run later programmatically."
             )
-            readme_result = extract_xml(readme_response, "readme")
-            readme_path.write_text(readme_result)
-            click.echo(FeedbackManager.info_file_created(file="README.md"))
-            created_something = True
+            prompt_result = create_resources_from_prompt(
+                ctx_config,
+                project,
+                prompt,
+                feature="tb_create",
+                instructions=prompt_instructions,
+            )
+            result.extend(prompt_result)
+            if prompt_result:
+                created_something = True
 
         if data or prompt:
             click.echo(FeedbackManager.success(message="✓ Resources created!\n"))
@@ -180,24 +165,30 @@ def create(
                     datasource_name = datasource_path.stem
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
-                    if has_json_path:
-                        mock_data = create_mock_data(
-                            datasource_name,
-                            datasource_content,
-                            rows,
-                            prompt,
-                            config,
-                            ctx_config,
-                            user_token,
-                            tb_client,
-                            format_="ndjson",
-                            folder=project.folder,
-                        )
-                        if mock_data:
-                            persist_fixture(datasource_name, mock_data, folder, format="ndjson")
-                            click.echo(FeedbackManager.info_file_created(file=f"fixtures/{datasource_name}.ndjson"))
-                            click.echo(FeedbackManager.success(message="✓ Done!"))
-                            created_something = True
+                    if not has_json_path:
+                        continue
+
+                    fixture_path = Path(folder) / "fixtures" / f"{datasource_name}.ndjson"
+                    fixture_existed = fixture_path.exists()
+                    fixture_prompt = (
+                        f"Generate {rows} rows of representative sample data for the Tinybird datasource defined in {datasource_path}. "
+                        f"Store the data in ndjson format at fixtures/{datasource_name}.ndjson."
+                    )
+                    if prompt.strip():
+                        fixture_prompt += f"\n\nOriginal project request:\n{prompt.strip()}"
+
+                    run_agent(
+                        ctx_config,
+                        project,
+                        True,
+                        prompt=fixture_prompt,
+                        feature="tb_mock",
+                    )
+
+                    if fixture_path.exists() and not fixture_existed:
+                        click.echo(FeedbackManager.info_file_created(file=f"fixtures/{datasource_name}.ndjson"))
+                        click.echo(FeedbackManager.success(message="✓ Done!"))
+                        created_something = True
 
         if not created_something and not len(result) > 0:
             click.echo(FeedbackManager.warning(message="△ No resources created\n"))
@@ -291,86 +282,39 @@ def create_project_structure(folder: str):
 
 
 def create_resources_from_prompt(
-    tb_client: TinyB,
-    user_token: str,
-    prompt: str,
+    config: Dict[str, Any],
     project: Project,
-    feature: str = "tb_create_resources",
+    prompt: str,
+    feature: str = "tb_create",
+    instructions: Optional[str] = None,
 ) -> List[Path]:
-    result: List[Path] = []
-    datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
-    pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
-    connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
-    resources_xml = "\n".join(
-        [
-            f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
-            for resource_type, resource_name, resource_content in [
-                ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
-            ]
-            + [
-                (
-                    "pipe",
-                    pipe.stem,
-                    pipe.read_text(),
-                )
-                for pipe in pipes_paths
-            ]
-            + [
-                (
-                    "connection",
-                    conn.stem,
-                    conn.read_text(),
-                )
-                for conn in connections_paths
-            ]
-        ]
-    )
-    llm = LLM(user_token=user_token, host=tb_client.host)
-    prompt_result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt, feature=feature)
-    prompt_result = extract_xml(prompt_result, "response")
-    resources = parse_xml(prompt_result, "resource")
-    datasources = []
-    pipes = []
-    connections = []
-    for resource_xml in resources:
-        resource_type = extract_xml(resource_xml, "type")
-        name = extract_xml(resource_xml, "name")
-        content = extract_xml(resource_xml, "content")
-        resource = {
-            "name": name,
-            "content": content,
-        }
-        if resource_type.lower() == "datasource":
-            datasources.append(resource)
-        elif resource_type.lower() == "pipe":
-            pipes.append(resource)
-        elif resource_type.lower() == "connection":
-            connections.append(resource)
-
-    for ds in datasources:
-        content = ds["content"].replace("```", "")
-        filename = f"{ds['name']}.datasource"
-        ds_file = generate_datafile(
-            content,
-            filename=filename,
-            data=None,
-            _format="ndjson",
-            force=True,
-            folder=project.folder,
-        )
-        result.append(ds_file)
-    for pipe in pipes:
-        content = pipe["content"].replace("```", "")
-        pipe_file = generate_pipe_file(pipe["name"], content, project.folder)
-        result.append(pipe_file)
+    """Run the agent in prompt mode and report newly created project resources."""
 
-    for conn in connections:
-        content = conn["content"].replace("```", "")
-        filename = f"{conn['name']}.connection"
-        conn_file = generate_connection_file(conn["name"], content, project.folder)
-        result.append(conn_file)
+    agent_prompt = prompt.strip()
+    if instructions:
+        instructions = instructions.strip()
+        if agent_prompt:
+            agent_prompt = f"{instructions}\n\n{agent_prompt}"
+        else:
+            agent_prompt = instructions
 
-    return result
+    if not agent_prompt:
+        return []
+
+    resources_before = _collect_project_resource_paths(project)
+    run_agent(config, project, True, prompt=agent_prompt, feature=feature)
+    resources_after = _collect_project_resource_paths(project)
+
+    created_resources = [Path(path) for path in sorted(resources_after - resources_before)]
+    return created_resources
+
+
+def _collect_project_resource_paths(project: Project) -> Set[Path]:
+    resources: Set[Path] = set()
+    resources.update(Path(path) for path in project.get_datasource_files())
+    resources.update(Path(path) for path in project.get_pipe_files())
+    resources.update(Path(path) for path in project.get_connection_files())
+    return resources
 
 
 def init_git(folder: str):
@@ -513,28 +457,6 @@ def save_context(prompt: str, feedback: str):
     context_file.write_text(f"- {prompt}\n{feedback}")
 
 
-def get_resources_xml(project: Project) -> str:
-    datasource_paths = [Path(f) for f in project.get_datasource_files()]
-    pipes_paths = [Path(f) for f in project.get_pipe_files()]
-    resources_xml = "\n".join(
-        [
-            f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
-            for resource_type, resource_name, resource_content in [
-                ("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
-            ]
-            + [
-                (
-                    "pipe",
-                    pipe.stem,
-                    pipe.read_text(),
-                )
-                for pipe in pipes_paths
-            ]
-        ]
-    )
-    return resources_xml
-
-
 def create_resources_from_data(
     data: str,
     project: Project,

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/datasource.py

@@ -768,13 +768,31 @@ def datasource_create(
 
     if datasource_type == "prompt":
         click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
-        user_token = config.get("user_token")
-        if not user_token:
+        if not config.get("user_token"):
             raise Exception("This action requires authentication. Run 'tb login' first.")
-        project_config = CLIConfig.get_project_config()
-        tb_client: TinyB = project_config.get_client(token=config.get("token"), host=config.get("host"))
-        create_resources_from_prompt(tb_client, user_token, prompt, project, feature="tb_datasource_create")
-        click.echo(FeedbackManager.success(message="✓ .datasource created!"))
+
+        instructions = (
+            "Create or update a Tinybird datasource (.datasource file) for this project. "
+            "Do not generate mock data or append data; those steps will run later programmatically."
+        )
+        if name:
+            instructions += f" Name the datasource '{name}'."
+
+        created_resources = create_resources_from_prompt(
+            config,
+            project,
+            prompt,
+            feature="tb_datasource_create",
+            instructions=instructions,
+        )
+        if any(path.suffix == ".datasource" for path in created_resources):
+            click.echo(FeedbackManager.success(message="✓ .datasource created!"))
+        else:
+            click.echo(
+                FeedbackManager.gray(
+                    message="△ No new datasource file detected. Existing resources may have been updated instead."
+                )
+            )
         return
 
     connection_required = datasource_type in ("kafka", "s3", "gcs")

tinybird-0.0.1.dev302/tinybird/tb/modules/mock.py

@@ -0,0 +1,41 @@
+import click
+
+from tinybird.tb.modules.agent import run_agent
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.exceptions import CLIMockException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.project import Project
+
+
+@cli.command()
+@click.argument("datasource", type=str)
+@click.option("--rows", type=int, default=10, help="Number of events to send")
+@click.option(
+    "--prompt",
+    type=str,
+    default="",
+    help="Extra context to use for data generation",
+)
+@click.option(
+    "--format",
+    "format_",
+    type=click.Choice(["ndjson", "csv"], case_sensitive=False),
+    default="ndjson",
+    help="Format of the fixture to create",
+)
+@click.pass_context
+def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
+    """Generate sample data for a data source."""
+
+    try:
+        project: Project = ctx.ensure_object(dict)["project"]
+        ctx_config = ctx.ensure_object(dict)["config"]
+        prompt = f"""Generate mock data for the following datasource: {datasource} with {rows} rows and {format_} format. Extra context: {prompt}"""
+        env = ctx.ensure_object(dict)["env"]
+        if env == "cloud":
+            prompt += "Append the fixture data to the datasource in Tinybird Cloud."
+
+        run_agent(ctx_config, project, True, prompt=prompt, feature="tb_mock")
+
+    except Exception as e:
+        raise CLIMockException(FeedbackManager.error(message=str(e)))

tinybird-0.0.1.dev302/tinybird/tb/modules/mock_common.py

@@ -0,0 +1,17 @@
+from tinybird.tb.client import TinyB
+from tinybird.tb.modules.common import push_data
+
+
+def append_mock_data(
+    tb_client: TinyB,
+    datasource_name: str,
+    url: str,
+):
+    push_data(
+        tb_client,
+        datasource_name,
+        url,
+        mode="append",
+        concurrency=1,
+        silent=True,
+    )

{tinybird-0.0.1.dev301 → tinybird-0.0.1.dev302}/tinybird/tb/modules/test.py

@@ -8,9 +8,10 @@ from typing import Any, Tuple
 import click
 
 from tinybird.tb.client import TinyB
+from tinybird.tb.modules.agent import run_agent
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.project import Project
-from tinybird.tb.modules.test_common import create_test, run_tests, update_test
+from tinybird.tb.modules.test_common import run_tests, update_test
 
 
 @cli.group()
@@ -24,18 +25,18 @@ def test(ctx: click.Context) -> None:
     help="Create a test for an existing pipe",
 )
 @click.argument("name_or_filename", type=str)
-@click.option(
-    "--prompt", type=str, default="Create a test for the selected pipe", help="Prompt to be used to create the test"
-)
+@click.option("--prompt", type=str, default="", help="Prompt to be used to create the test")
 @click.pass_context
 def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
     """
     Create a test for an existing pipe
     """
     project: Project = ctx.ensure_object(dict)["project"]
-    client: TinyB = ctx.ensure_object(dict)["client"]
     config: dict[str, Any] = ctx.ensure_object(dict)["config"]
-    create_test(name_or_filename, prompt, project, client, config=config)
+    prompt = (
+        f"""Create tests for the following pipe: {name_or_filename}. Extra context: {prompt or "No extra context."}"""
+    )
+    run_agent(config, project, True, prompt=prompt, feature="tb_test_create")
 
 
 @test.command(