tinybird 0.0.1.dev242__tar.gz → 0.0.1.dev244__tar.gz

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev242
+Version: 0.0.1.dev244
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev242'
-__revision__ = '62ad714'
+__version__ = '0.0.1.dev244'
+__revision__ = '434c7ff'

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/tinybird/tb/modules/agent/agent.py

@@ -1,5 +1,8 @@
 import sys
+import uuid
 from datetime import datetime
+from functools import partial
+from typing import Any
 
 import click
 from prompt_toolkit import prompt
@@ -26,20 +29,29 @@ from tinybird.tb.modules.agent.animations import ThinkingAnimation
 from tinybird.tb.modules.agent.banner import display_banner
 from tinybird.tb.modules.agent.memory import clear_history, load_history
 from tinybird.tb.modules.agent.models import create_model
-from tinybird.tb.modules.agent.prompts import datafile_instructions, plan_instructions, sql_instructions
+from tinybird.tb.modules.agent.prompts import (
+    datafile_instructions,
+    plan_instructions,
+    resources_prompt,
+    sql_instructions,
+)
 from tinybird.tb.modules.agent.tools.create_datafile import create_datafile
 from tinybird.tb.modules.agent.tools.explore import explore_data
 from tinybird.tb.modules.agent.tools.plan import plan
 from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
 from tinybird.tb.modules.agent.utils import TinybirdAgentContext
+from tinybird.tb.modules.build_common import process as build_process
+from tinybird.tb.modules.exceptions import CLIBuildException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.project import Project
 
 
 class TinybirdAgent:
-    def __init__(self, token: str, host: str, folder: str):
+    def __init__(self, token: str, host: str, project: Project):
         self.token = token
         self.host = host
-        self.folder = folder
+        self.project = project
         self.messages: list[ModelMessage] = []
         self.agent = Agent(
             model=create_model(token, host),
@@ -83,9 +95,10 @@ You have access to the following tools:
 4. Without asking, use the `create_datafile` tool to create the datafile, because it will ask for confirmation before creating the file.
 5. Check the result of the `create_datafile` tool to see if the datafile was created successfully.
 6. If the datafile was created successfully, report the result to the user.
-7. If the datafile was not created successfully, finish the process and just wait for a new user prompt.
+7. If the datafile was not created, finish the process and just wait for a new user prompt.
+8. If the datafile was created successfully, but the built failed, try to fix the error and repeat the process.
 
-IMPORTANT: If the user cancels some of the steps or there is an error, DO NOT continue with the plan. Stop the process and wait for the user before using any other tool.
+IMPORTANT: If the user cancels some of the steps or there is an error in file creation, DO NOT continue with the plan. Stop the process and wait for the user before using any other tool.
 
 # When planning the creation or update of resources:
 {plan_instructions}
@@ -134,9 +147,10 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
         """Keep only the last 5 messages to manage token usage."""
         return self.messages[-5:] if len(self.messages) > 5 else self.messages
 
-    def run(self, user_prompt: str) -> None:
+    def run(self, user_prompt: str, project: Project) -> None:
+        user_prompt = f"{user_prompt}\n\n# Existing resources in the project:\n{resources_prompt(project)}"
         client = TinyB(token=self.token, host=self.host)
-
+        folder = self.project.folder
         thinking_animation = ThinkingAnimation(message="Chirping", delay=0.15)
         thinking_animation.start()
 
@@ -145,7 +159,9 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
             deps=TinybirdAgentContext(
                 # context does not support the whole client, so we need to pass only the functions we need
                 explore_data=client.explore_data,
-                folder=self.folder,
+                build_project=partial(build_project, folder=folder),
+                get_project_files=project.get_project_files,
+                folder=folder,
                 thinking_animation=thinking_animation,
             ),
             message_history=self.messages,
@@ -158,11 +174,13 @@ Today is {datetime.now().strftime("%Y-%m-%d")}
         click.echo("\n")
 
 
-def run_agent(token: str, host: str, folder: str):
+def run_agent(config: dict[str, Any], project: Project):
     display_banner()
 
     try:
-        agent = TinybirdAgent(token, host, folder)
+        token = config["token"]
+        host = config["host"]
+        agent = TinybirdAgent(token, host, project)
         click.echo()
         click.echo(FeedbackManager.success(message="Welcome to Tinybird Code"))
         click.echo(FeedbackManager.info(message="Describe what you want to create and I'll help you build it"))
@@ -207,7 +225,7 @@ def run_agent(token: str, host: str, folder: str):
                 elif user_input.strip() == "":
                     continue
                 else:
-                    agent.run(user_input)
+                    agent.run(user_input, project)
 
             except KeyboardInterrupt:
                 click.echo(FeedbackManager.info(message="Goodbye!"))
@@ -219,3 +237,12 @@ def run_agent(token: str, host: str, folder: str):
     except Exception as e:
         click.echo(FeedbackManager.error(message=f"Error: {e}"))
         sys.exit(1)
+
+
+def build_project(folder: str) -> None:
+    workspace_name = f"tmp_workspace_{uuid.uuid4()}"
+    project = Project(folder, workspace_name=workspace_name)
+    local_client = get_tinybird_local_client({"path": folder, "name": workspace_name}, test=True, silent=True)
+    build_error = build_process(project=project, tb_client=local_client, watch=False, silent=True, exit_on_error=False)
+    if build_error:
+        raise CLIBuildException(build_error)

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/tinybird/tb/modules/agent/prompts.py

@@ -1,3 +1,7 @@
+from pathlib import Path
+
+from tinybird.tb.modules.project import Project
+
 plan_instructions = """
 When asked to create a plan, you MUST respond with this EXACT format and NOTHING ELSE:
 
@@ -81,3 +85,35 @@ datafile_instructions = """
 - Datasource files will be created under the `/datasources` folder.
 </datafile_instructions>
 """
+
+
+def resources_prompt(project: Project) -> str:
+    files = project.get_project_files()
+
+    if not files:
+        return "No resources found"
+
+    paths = [Path(file_path) for file_path in files]
+
+    def get_resource_type(path: Path) -> str:
+        if path.suffix.lower() == ".pipe":
+            return Project.get_pipe_type(str(path))
+        elif path.suffix.lower() == ".datasource":
+            return "datasource"
+        elif path.suffix.lower() == ".connection":
+            return "connection"
+        return "unknown"
+
+    return "\n".join(
+        [
+            f"""
+<resource>
+    <path>{file_path.relative_to(project.folder)}</path>
+    <type>{get_resource_type(file_path)}</type>
+    <name>{file_path.stem}</name>
+    <content>{file_path.read_text()}</content>
+</resource>
+"""
+            for file_path in paths
+        ]
+    )

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/tinybird/tb/modules/agent/tools/create_datafile.py

@@ -4,13 +4,16 @@ import click
 from pydantic_ai import RunContext
 
 from tinybird.tb.modules.agent.utils import Datafile, TinybirdAgentContext, show_options
+from tinybird.tb.modules.exceptions import CLIBuildException
+from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-def get_resource_confirmation(resource: Datafile) -> bool:
+def get_resource_confirmation(resource: Datafile, exists: bool) -> bool:
     """Get user confirmation for creating a resource"""
     while True:
+        action = "create" if not exists else "update"
         result = show_options(
-            options=[f"Yes, create {resource.type} '{resource.name}'", "No, and tell Tinybird Code what to do"],
+            options=[f"Yes, {action} {resource.type} '{resource.name}'", "No, and tell Tinybird Code what to do"],
             title=f"What would you like to do with {resource.type} '{resource.name}'?",
         )
 
@@ -36,29 +39,28 @@ def create_datafile(ctx: RunContext[TinybirdAgentContext], resource: Datafile) -
     """
     try:
         ctx.deps.thinking_animation.stop()
-        click.echo()
         click.echo(resource.content)
-        confirmation = get_resource_confirmation(resource)
+        resource.pathname = resource.pathname.removeprefix("/")
+        path = Path(ctx.deps.folder) / resource.pathname
+        exists = str(path) in ctx.deps.get_project_files()
+        confirmation = get_resource_confirmation(resource, exists)
         ctx.deps.thinking_animation.start()
 
         if not confirmation:
             return f"Resource {resource.pathname} was not created. User cancelled creation."
 
-        resource.pathname = resource.pathname.removeprefix("/")
-
-        path = Path(ctx.deps.folder) / resource.pathname
-
         folder_path = path.parent
-
-        if not folder_path.exists():
-            folder_path.mkdir()
-
-        if not path.exists():
-            path.touch()
+        folder_path.mkdir(parents=True, exist_ok=True)
+        path.touch(exist_ok=True)
 
         path.write_text(resource.content)
-
+        ctx.deps.build_project()
         return f"Created {resource.pathname}"
 
+    except CLIBuildException as e:
+        ctx.deps.thinking_animation.stop()
+        click.echo(FeedbackManager.error(message=e))
+        ctx.deps.thinking_animation.start()
+        return f"Error building project: {e}"
     except Exception as e:
         return f"Error creating {resource.pathname}: {e}"

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/tinybird/tb/modules/agent/tools/plan.py

@@ -34,7 +34,6 @@ def plan(ctx: RunContext[TinybirdAgentContext], plan: str) -> str:
     """
     try:
         ctx.deps.thinking_animation.stop()
-        click.echo()
         click.echo(plan)
         confirmation = get_plan_confirmation()
         ctx.deps.thinking_animation.start()

{tinybird-0.0.1.dev242 → tinybird-0.0.1.dev244}/tinybird/tb/modules/agent/utils.py

@@ -20,7 +20,9 @@ from pydantic import BaseModel, Field
 class TinybirdAgentContext(BaseModel):
     explore_data: Callable[[str], str]
     folder: str
+    build_project: Callable[[], None]
     thinking_animation: Any
+    get_project_files: Callable[[], List[str]]
 
 
 default_style = Style.from_dict(

tinybird-0.0.1.dev244/tinybird/tb/modules/build.py

@@ -0,0 +1,205 @@
+import threading
+import time
+from copy import deepcopy
+from functools import partial
+from pathlib import Path
+from typing import Callable, List
+from urllib.parse import urlencode
+
+import click
+
+import tinybird.context as context
+from tinybird.datafile.exceptions import ParseException
+from tinybird.datafile.parse_datasource import parse_datasource
+from tinybird.datafile.parse_pipe import parse_pipe
+from tinybird.tb.client import TinyB
+from tinybird.tb.modules.build_common import process
+from tinybird.tb.modules.cli import cli
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.datafile.playground import folder_playground
+from tinybird.tb.modules.dev_server import BuildStatus, start_server
+from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.project import Project
+from tinybird.tb.modules.secret_common import load_secrets
+from tinybird.tb.modules.shell import Shell, print_table_formatted
+from tinybird.tb.modules.watch import watch_files, watch_project
+
+
+@cli.command()
+@click.option("--watch", is_flag=True, default=False, help="Watch for changes and rebuild automatically")
+@click.pass_context
+def build(ctx: click.Context, watch: bool) -> None:
+    """
+    Validate and build the project server side.
+    """
+    project: Project = ctx.ensure_object(dict)["project"]
+    tb_client: TinyB = ctx.ensure_object(dict)["client"]
+
+    if project.has_deeper_level():
+        click.echo(
+            FeedbackManager.warning(
+                message="Your project contains directories nested deeper than the default scan depth (max_depth=3). "
+                "Files in these deeper directories will not be processed. "
+                "To include all nested directories, run `tb --max-depth <depth> <cmd>` with a higher depth value."
+            )
+        )
+
+    load_secrets(project, tb_client)
+    click.echo(FeedbackManager.highlight_building_project())
+    process(project=project, tb_client=tb_client, watch=False)
+    if watch:
+        run_watch(
+            project=project,
+            tb_client=tb_client,
+            process=partial(process, project=project, tb_client=tb_client, watch=True),
+        )
+
+
+@cli.command("dev", help="Build the project server side and watch for changes.")
+@click.option("--data-origin", type=str, default="", help="Data origin: local or cloud")
+@click.option("--ui", is_flag=True, default=False, help="Connect your local project to Tinybird UI")
+@click.pass_context
+def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
+    if data_origin == "cloud":
+        return dev_cloud(ctx)
+    project: Project = ctx.ensure_object(dict)["project"]
+    tb_client: TinyB = ctx.ensure_object(dict)["client"]
+    build_status = BuildStatus()
+    if ui:
+        server_thread = threading.Thread(
+            target=start_server, args=(project, tb_client, process, build_status), daemon=True
+        )
+        server_thread.start()
+        # Wait for the server to start
+        time.sleep(0.5)
+
+    load_secrets(project, tb_client)
+    click.echo(FeedbackManager.highlight_building_project())
+    process(project=project, tb_client=tb_client, watch=True, build_status=build_status)
+    run_watch(
+        project=project,
+        tb_client=tb_client,
+        process=partial(process, project=project, tb_client=tb_client, build_status=build_status),
+    )
+
+
+def run_watch(project: Project, tb_client: TinyB, process: Callable) -> None:
+    shell = Shell(project=project, tb_client=tb_client, playground=False)
+    click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
+    watcher_thread = threading.Thread(
+        target=watch_project,
+        args=(shell, process, project),
+        daemon=True,
+    )
+    watcher_thread.start()
+    shell.run()
+
+
+def is_vendor(f: Path) -> bool:
+    return f.parts[0] == "vendor"
+
+
+def get_vendor_workspace(f: Path) -> str:
+    return f.parts[1]
+
+
+def is_endpoint(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f) and f.parts[0] == "endpoints"
+
+
+def is_pipe(f: Path) -> bool:
+    return f.suffix == ".pipe" and not is_vendor(f)
+
+
+def check_filenames(filenames: List[str]):
+    parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
+    incl_suffix = ".incl"
+
+    for filename in filenames:
+        file_suffix = Path(filename).suffix
+        if file_suffix == incl_suffix:
+            continue
+
+        parser = parser_matrix.get(file_suffix)
+        if not parser:
+            raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
+
+        parser(filename)
+
+
+def dev_cloud(
+    ctx: click.Context,
+) -> None:
+    project: Project = ctx.ensure_object(dict)["project"]
+    config = CLIConfig.get_project_config()
+    tb_client: TinyB = config.get_client()
+    context.disable_template_security_validation.set(True)
+
+    def process(filenames: List[str], watch: bool = False):
+        datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
+        if len(datafiles) > 0:
+            check_filenames(filenames=datafiles)
+            folder_playground(
+                project, config, tb_client, filenames=datafiles, is_internal=False, current_ws=None, local_ws=None
+            )
+        if len(filenames) > 0 and watch:
+            filename = filenames[0]
+            build_and_print_resource(config, tb_client, filename)
+
+    datafiles = project.get_project_files()
+    filenames = datafiles
+
+    def build_once(filenames: List[str]):
+        ok = False
+        try:
+            click.echo(FeedbackManager.highlight(message="» Building project...\n"))
+            time_start = time.time()
+            process(filenames=filenames, watch=False)
+            time_end = time.time()
+            elapsed_time = time_end - time_start
+
+            click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
+            ok = True
+        except Exception as e:
+            error_path = Path(".tb_error.txt")
+            if error_path.exists():
+                content = error_path.read_text()
+                content += f"\n\n{str(e)}"
+                error_path.write_text(content)
+            else:
+                error_path.write_text(str(e))
+            click.echo(FeedbackManager.error_exception(error=e))
+            ok = False
+        return ok
+
+    build_ok = build_once(filenames)
+
+    shell = Shell(project=project, tb_client=tb_client, playground=True)
+    click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
+    watcher_thread = threading.Thread(
+        target=watch_files, args=(filenames, process, shell, project, build_ok), daemon=True
+    )
+    watcher_thread.start()
+    shell.run()
+
+
+def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename: str):
+    resource_path = Path(filename)
+    name = resource_path.stem
+    playground_name = name if filename.endswith(".pipe") else None
+    user_client = deepcopy(tb_client)
+    user_client.token = config.get_user_token() or ""
+    cli_params = {}
+    cli_params["workspace_id"] = config.get("id", None)
+    data = user_client._req(f"/v0/playgrounds?{urlencode(cli_params)}")
+    playgrounds = data["playgrounds"]
+    playground = next((p for p in playgrounds if p["name"] == (f"{playground_name}" + "__tb__playground")), None)
+    if not playground:
+        return
+    playground_id = playground["id"]
+    last_node = playground["nodes"][-1]
+    if not last_node:
+        return
+    node_sql = last_node["sql"]
+    res = tb_client.query(f"{node_sql} FORMAT JSON", playground=playground_id)
+    print_table_formatted(res, name)