tinybird 0.0.1.dev43__py3-none-any.whl → 0.0.1.dev44__py3-none-any.whl

tinybird/prompts.py

@@ -397,9 +397,19 @@ Follow the instructions and generate the following response with no additional t
 """
 
 
-def create_prompt(existing_resources: str) -> str:
+def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
+    feedback_history = ""
+    if feedback and history:
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
+Use the following feedback and history to regenerate the response:
+Feedback to improve the response:
+{feedback}
+History of previous results:
+{history}"""
+
     return """
-You are a Tinybird expert. You will be given a prompt to generate Tinybird resources: datasources and/or pipes.
+You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
 <existing_resources>{existing_resources}</existing_resources>
 {datasource_instructions}
 {pipe_instructions}
@@ -408,6 +418,9 @@ You are a Tinybird expert. You will be given a prompt to generate Tinybird resou
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+
+{feedback_history}
+
 Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
 <response>
     <resource>
@@ -426,10 +439,21 @@ Use the following format to generate the response and do not wrap it in any othe
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        feedback_history=feedback_history,
     )
 
 
-def mock_prompt(rows: int) -> str:
+def mock_prompt(rows: int, feedback: str = "", history: str = "") -> str:
+    feedback_history = ""
+    if feedback and history:
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
+Use the following feedback and history to regenerate the response:
+Feedback to improve the response:
+{feedback}
+History of previous results:
+{history}"""
+
     return f"""
 Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
 
@@ -570,6 +594,9 @@ Follow the instructions and generate the following response with no additional t
 <response>
     <sql>[raw sql query here]</sql>
 </response>
+
+{feedback_history}
+
 """
 
 

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev43'
-__revision__ = 'ecb1311'
+__version__ = '0.0.1.dev44'
+__revision__ = '637cb4c'

tinybird/tb/cli.py

@@ -6,7 +6,6 @@ if sys.platform == "win32":
 
 import tinybird.tb.modules.auth
 import tinybird.tb.modules.build
-import tinybird.tb.modules.build_client
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
 import tinybird.tb.modules.copy
@@ -18,6 +17,7 @@ import tinybird.tb.modules.fmt
 import tinybird.tb.modules.job
 import tinybird.tb.modules.local
 import tinybird.tb.modules.login
+import tinybird.tb.modules.materialization
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag

tinybird/tb/modules/build.py

@@ -12,6 +12,7 @@ import requests
 from tinybird.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
@@ -37,9 +38,10 @@ def build(ctx: click.Context, watch: bool) -> None:
         if file_changed and file_changed.endswith(".ndjson"):
             rebuild_fixture(project, tb_client, file_changed)
         else:
-            build_project(project, tb_client)
+            build_project(project, tb_client, file_changed)
             try:
                 if file_changed:
+                    asyncio.run(folder_build(project, filenames=[file_changed]))
                     build_and_print_resource(tb_client, file_changed, diff)
             except Exception:
                 pass
@@ -61,7 +63,7 @@ def build(ctx: click.Context, watch: bool) -> None:
         shell.run()
 
 
-def build_project(project: Project, tb_client: TinyB) -> None:
+def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str] = None) -> None:
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
@@ -103,20 +105,23 @@ def build_project(project: Project, tb_client: TinyB) -> None:
         if build_result == "success":
             datasources = result.get("datasources", [])
             pipes = result.get("pipes", [])
-            for ds in datasources:
-                ds_path_str: Optional[str] = next(
-                    (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
-                )
-                if ds_path_str:
-                    ds_path = Path(ds_path_str)
-                    ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
-            for pipe in pipes:
-                pipe_name = pipe.get("name")
-                pipe_path_str: Optional[str] = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
-                if pipe_path_str:
-                    pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
+            if not file_changed:
+                for ds in datasources:
+                    ds_path_str: Optional[str] = next(
+                        (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
+                    )
+                    if ds_path_str:
+                        ds_path = Path(ds_path_str)
+                        ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
+                        click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
+                for pipe in pipes:
+                    pipe_name = pipe.get("name")
+                    pipe_path_str: Optional[str] = next(
+                        (p for p in project_files if p.endswith(pipe_name + ".pipe")), None
+                    )
+                    if pipe_path_str:
+                        pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
+                        click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
 
             try:
                 for filename in project_files:

tinybird/tb/modules/cli.py

@@ -6,10 +6,9 @@
 import json
 import logging
 import os
-import pprint
 from os import getcwd
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import click
 import humanfriendly
@@ -35,13 +34,7 @@ from tinybird.tb.modules.common import (
 )
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import build_graph
-from tinybird.tb.modules.datafile.common import Datafile, DatafileSyntaxError
 from tinybird.tb.modules.datafile.diff import diff_command
-from tinybird.tb.modules.datafile.exceptions import (
-    ParseException,
-)
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
@@ -132,55 +125,7 @@ async def cli(
     ctx.ensure_object(dict)["project"] = project
 
 
-@cli.command(hidden=True)
-@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
-@click.option("--debug", is_flag=True, default=False, help="Print internal representation")
-@click.pass_context
-def check(ctx: Context, filenames: List[str], debug: bool) -> None:
-    """Check file syntax."""
-
-    if not filenames:
-        project: Project = ctx.ensure_object(dict)["project"]
-        filenames = project.get_project_files()
-
-    def process(filenames: Iterable):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
-        incl_suffix = ".incl"
-        try:
-            for filename in filenames:
-                if os.path.isdir(filename):
-                    process(filenames=filename)
-
-                click.echo(FeedbackManager.info_processing_file(filename=filename))
-
-                file_suffix = Path(filename).suffix
-                if file_suffix == incl_suffix:
-                    click.echo(FeedbackManager.info_ignoring_incl_file(filename=filename))
-                    continue
-
-                doc: Datafile
-                parser = parser_matrix.get(file_suffix)
-                if not parser:
-                    raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
-
-                doc = parser(filename)
-
-                click.echo(FeedbackManager.success_processing_file(filename=filename))
-                if debug:
-                    pp = pprint.PrettyPrinter()
-                    for x in doc.nodes:
-                        pp.pprint(x)
-
-        except DatafileSyntaxError as e:
-            # TODO(eclbg): add the filename to the error message
-            raise CLIException(str(e))
-        except ParseException as e:
-            raise CLIException(FeedbackManager.error_exception(error=e))
-
-    process(filenames=filenames)
-
-
-@cli.command(hidden=True)
+@cli.command()
 @click.option(
     "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
 )

tinybird/tb/modules/common.py

@@ -163,7 +163,7 @@ def generate_datafile(
     force: Optional[bool] = False,
     _format: Optional[str] = "csv",
     folder: Optional[str] = None,
-):
+) -> Path:
     p = Path(filename)
     base = Path("datasources")
     if folder:
@@ -190,6 +190,7 @@ def generate_datafile(
                     fixture_file.write(data[: data.rfind(newline)])
     else:
         click.echo(FeedbackManager.error_file_already_exists(file=f))
+    return f
 
 
 async def get_current_workspace(config: CLIConfig) -> Optional[Dict[str, Any]]:

tinybird/tb/modules/copy.py

@@ -18,7 +18,7 @@ from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-@cli.group(hidden=True)
+@cli.group()
 @click.pass_context
 def copy(ctx):
     """Copy pipe commands"""

tinybird/tb/modules/create.py

@@ -18,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
 from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
 from tinybird.tb.modules.local_common import get_tinybird_local_client
+from tinybird.tb.modules.project import Project
 
 
 @cli.command()
@@ -41,8 +42,10 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
 )
 @click.option("--rows", type=int, default=10, help="Number of events to send")
 @click.option("--cursor", default=False, is_flag=True, help="Create .cursorrules file with Tinybird rules")
+@click.pass_context
 @coro
 async def create(
+    ctx: click.Context,
     data: Optional[str],
     prompt: Optional[str],
     folder: Optional[str],
@@ -50,13 +53,14 @@ async def create(
     cursor: bool,
 ) -> None:
     """Initialize a new project."""
+    project: Project = ctx.ensure_object(dict)["project"]
     folder = folder or getcwd()
     folder_path = Path(folder)
     if not folder_path.exists():
         folder_path.mkdir()
 
     try:
-        config = CLIConfig.get_project_config(folder)
+        config = CLIConfig.get_project_config(str(project.path))
         tb_client = config.get_client()
         user_token: Optional[str] = None
         if prompt:
@@ -80,7 +84,7 @@ async def create(
             click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
 
         click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-        datasources_created = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
+        result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
         click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
         if not already_has_cicd(folder):
@@ -89,7 +93,7 @@ async def create(
             await init_cicd(data_project_dir=os.path.relpath(folder))
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
-        if validate_fixtures(folder) and datasources_created:
+        if should_generate_fixtures(result):
             click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
 
             if data:
@@ -110,7 +114,7 @@ async def create(
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
                     if has_json_path:
-                        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
+                        prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
                         response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
                         sql = extract_xml(response, "sql")
                         sql = sql.split("FORMAT")[0]
@@ -139,9 +143,10 @@ def validate_project_structure(folder: str) -> bool:
     return all((Path(folder) / path).exists() for path in PROJECT_PATHS)
 
 
-def validate_fixtures(folder: str) -> bool:
-    datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
-    return len(datasource_files) > 0
+def should_generate_fixtures(result: str) -> bool:
+    if not result:
+        return False
+    return "<type>datasource</type>" in result
 
 
 def already_has_cicd(folder: str) -> bool:
@@ -168,12 +173,12 @@ async def create_resources(
     prompt: Optional[str],
     folder: str,
 ):
-    force = True
+    result = ""
     folder_path = Path(folder)
     if data:
         path = folder_path / data
         format = path.suffix.lstrip(".")
-        await _generate_datafile(str(path), local_client, format=format, force=force)
+        await _generate_datafile(str(path), local_client, format=format, force=True)
         name = data.split(".")[0]
         generate_pipe_file(
             f"{name}_endpoint",
@@ -185,7 +190,10 @@ TYPE ENDPOINT
             """,
             folder,
         )
-        return True
+        result = (
+            f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
+        )
+
     elif prompt and user_token:
         datasource_paths = [
             Path(folder) / "datasources" / f
@@ -212,41 +220,74 @@ TYPE ENDPOINT
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)
-        result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
-        result = extract_xml(result, "response")
-        resources = parse_xml(result, "resource")
-        datasources = []
-        pipes = []
-        for resource_xml in resources:
-            resource_type = extract_xml(resource_xml, "type")
-            name = extract_xml(resource_xml, "name")
-            content = extract_xml(resource_xml, "content")
-            resource = {
-                "name": name,
-                "content": content,
-            }
-            if resource_type.lower() == "datasource":
-                datasources.append(resource)
-            elif resource_type.lower() == "pipe":
-                pipes.append(resource)
-
-        for ds in datasources:
-            content = ds["content"].replace("```", "")
-            filename = f"{ds['name']}.datasource"
-            generate_datafile(
-                content,
-                filename=filename,
-                data=None,
-                _format="ndjson",
-                force=force,
-                folder=folder,
-            )
-
-        for pipe in pipes:
-            content = pipe["content"].replace("```", "")
-            generate_pipe_file(pipe["name"], content, folder)
-
-        return len(datasources) > 0
+        result = ""
+        iterations = 0
+        history = ""
+        generated_paths: list[Path] = []
+
+        while iterations < 10:
+            feedback = ""
+            if iterations > 0:
+                feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
+            if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
+                break
+            else:
+                if iterations > 0:
+                    click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
+                    for path in generated_paths:
+                        path.unlink()
+                    generated_paths = []
+
+                save_context(prompt, feedback)
+                result = llm.ask(system_prompt=create_prompt(resources_xml, feedback, history), prompt=prompt)
+                result = extract_xml(result, "response")
+                history = (
+                    history
+                    + f"""
+                <result_iteration_{iterations}>
+                    {result}
+                </result_iteration_{iterations}>
+                """
+                )
+                resources = parse_xml(result, "resource")
+                datasources = []
+                pipes = []
+                for resource_xml in resources:
+                    resource_type = extract_xml(resource_xml, "type")
+                    name = extract_xml(resource_xml, "name")
+                    content = extract_xml(resource_xml, "content")
+                    resource = {
+                        "name": name,
+                        "content": content,
+                    }
+                    if resource_type.lower() == "datasource":
+                        datasources.append(resource)
+                    elif resource_type.lower() == "pipe":
+                        pipes.append(resource)
+
+                for ds in datasources:
+                    content = ds["content"].replace("```", "")
+                    filename = f"{ds['name']}.datasource"
+                    datasource_path = generate_datafile(
+                        content,
+                        filename=filename,
+                        data=None,
+                        _format="ndjson",
+                        force=True,
+                        folder=folder,
+                    )
+                    generated_paths.append(datasource_path)
+                for pipe in pipes:
+                    content = pipe["content"].replace("```", "")
+                    pipe_path = generate_pipe_file(pipe["name"], content, folder)
+                    generated_paths.append(pipe_path)
+
+                iterations += 1
+
+        if iterations == 10:
+            click.echo(FeedbackManager.info(message="Too many iterations. Change the prompt and try again."))
+
+        return result
 
 
 def init_git(folder: str):
@@ -266,7 +307,7 @@ def init_git(folder: str):
         raise CLIException(f"Error initializing Git: {e}")
 
 
-def generate_pipe_file(name: str, content: str, folder: str):
+def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     def is_copy(content: str) -> bool:
         return re.search(r"TYPE copy", content, re.IGNORECASE) is not None
 
@@ -292,6 +333,7 @@ def generate_pipe_file(name: str, content: str, folder: str):
     with open(f"{f}", "w") as file:
         file.write(content)
     click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
+    return f.relative_to(folder)
 
 
 def already_has_cursorrules(folder: str) -> bool:
@@ -301,3 +343,20 @@ def already_has_cursorrules(folder: str) -> bool:
 def create_cursorrules(folder: str):
     cursorrules_file = Path(folder) / ".cursorrules"
     cursorrules_file.write_text(cursorrules_prompt)
+
+
+def get_context_file() -> Path:
+    context_file = Path(os.path.expanduser("~/.tb_create_context"))
+    if not context_file.exists():
+        context_file.touch()
+    return context_file
+
+
+def get_context() -> str:
+    context_file = get_context_file()
+    return context_file.read_text()
+
+
+def save_context(prompt: str, feedback: str):
+    context_file = get_context_file()
+    context_file.write_text(f"- {prompt}\n{feedback}")