tinybird 0.0.1.dev42__tar.gz → 0.0.1.dev44__tar.gz

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev42
+Version: 0.0.1.dev44
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/client.py

@@ -72,7 +72,7 @@ def parse_error_response(response: Response) -> str:
         if content.get("error", None):
             error = content["error"]
             if content.get("errors", None):
-                error += f' -> errors: {content.get("errors")}'
+                error += f" -> errors: {content.get('errors')}"
         else:
             error = json.dumps(response, indent=4)
         return error

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/connectors.py

@@ -246,7 +246,7 @@ class BigQuery(Connector):
                     uri='{self.gcs.gs_url()}{destination}*.csv',
                     format='CSV',
                     overwrite=true,
-                    header={ "true" if with_headers else "false" },
+                    header={"true" if with_headers else "false"},
                     field_delimiter=',') AS
                     {sql}
                 """
@@ -319,7 +319,7 @@ class Snowflake(Connector):
 
     def create_stage(self):
         sql = f"""
-                    create stage "{self.options['schema']}".{self.stage()}
+                    create stage "{self.options["schema"]}".{self.stage()}
                         url='{self.gcs.gcs_url()}'
                         storage_integration = {self.storage_integration()};
                 """
@@ -337,7 +337,7 @@ class Snowflake(Connector):
                     from ({sql})
                     overwrite = true
                     file_format = (TYPE=CSV COMPRESSION=NONE ESCAPE_UNENCLOSED_FIELD=NONE FIELD_DELIMITER='|' FIELD_OPTIONALLY_ENCLOSED_BY='"' null_if=())
-                    header = {"true" if with_headers else "false" }
+                    header = {"true" if with_headers else "false"}
                     max_file_size = 2500000000;
                 """
         self.execute(sql)

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/datafile.py

@@ -2175,8 +2175,8 @@ class PipeCheckerRunner:
                             AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                             AND error = 0
                             AND not mapContains(parameters, '__tb__semver')
-                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
-                            { extra_where_clause }
+                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
+                            {extra_where_clause}
                         Limit 5000000 -- Enough to bring data while not processing all requests from highly used pipes
                         )
                     group by request_param_names, http_method
@@ -2202,7 +2202,7 @@ class PipeCheckerRunner:
                                         AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                                         AND error = 0
                                         AND not mapContains(parameters, '__tb__semver')
-                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
+                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
                                         {extra_where_clause}
                                     LIMIT {limit}
                                 )
@@ -4425,7 +4425,7 @@ async def folder_push(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -4442,7 +4442,7 @@ async def folder_push(
                     if raise_on_exists:
                         raise AlreadyExistsException(
                             FeedbackManager.warning_name_already_exists(
-                                name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
+                                name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
                             )
                         )
                     else:
@@ -4457,7 +4457,7 @@ async def folder_push(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -5013,7 +5013,7 @@ async def format_engine(
     else:
         if node.get("engine", None):
             empty = '""'
-            file_parts.append(f'ENGINE {node["engine"]["type"]}' if node.get("engine", {}).get("type") else empty)
+            file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
             file_parts.append(DATAFILE_NEW_LINE)
             for arg in sorted(node["engine"].get("args", [])):
                 elem = ", ".join([x.strip() for x in arg[1].split(",")])
@@ -5030,7 +5030,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
     if node_type == PipeNodeTypes.MATERIALIZED:
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
-        file_parts.append(f'DATASOURCE {node["datasource"]}')
+        file_parts.append(f"DATASOURCE {node['datasource']}")
         file_parts.append(DATAFILE_NEW_LINE)
         await format_engine(file_parts, node)
 
@@ -5038,10 +5038,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
     if node_type == PipeNodeTypes.COPY:
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
-        file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
+        file_parts.append(f"TARGET_DATASOURCE {node['target_datasource']}")
         if node.get("mode"):
             file_parts.append(DATAFILE_NEW_LINE)
-            file_parts.append(f'COPY_MODE {node.get("mode")}')
+            file_parts.append(f"COPY_MODE {node.get('mode')}")
 
         if node.get(CopyParameters.COPY_SCHEDULE):
             is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
@@ -5095,7 +5095,7 @@ async def format_node(
     if item and not unroll_includes:
         return
 
-    file_parts.append(f'NODE {node["name"].strip()}')
+    file_parts.append(f"NODE {node['name'].strip()}")
     file_parts.append(DATAFILE_NEW_LINE)
 
     from collections import namedtuple

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/feedback_manager.py

@@ -951,7 +951,7 @@ Ready? """
     )
     success_datasource_alter = success_message("** The Data Source has been correctly updated.")
     success_datasource_kafka_connected = success_message(
-        "** Data Source '{id}' created\n" "** Kafka streaming connection configured successfully!"
+        "** Data Source '{id}' created\n** Kafka streaming connection configured successfully!"
     )
     success_datasource_shared = success_message(
         "** The Data Source {datasource} has been correctly shared with {workspace}"

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/prompts.py

@@ -373,6 +373,7 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
     - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
     - If there are no parameters, you can omit parameters and generate a single test.
     - The format of the parameters is the following: ?param1=value1&param2=value2&param3=value3
+    - If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive.
 </instructions>
 
 This is an example of a test with parameters:
@@ -396,9 +397,19 @@ Follow the instructions and generate the following response with no additional t
 """
 
 
-def create_prompt(existing_resources: str) -> str:
+def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
+    feedback_history = ""
+    if feedback and history:
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
+Use the following feedback and history to regenerate the response:
+Feedback to improve the response:
+{feedback}
+History of previous results:
+{history}"""
+
     return """
-You are a Tinybird expert. You will be given a prompt to generate Tinybird resources: datasources and/or pipes.
+You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
 <existing_resources>{existing_resources}</existing_resources>
 {datasource_instructions}
 {pipe_instructions}
@@ -407,6 +418,9 @@ You are a Tinybird expert. You will be given a prompt to generate Tinybird resou
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+
+{feedback_history}
+
 Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
 <response>
     <resource>
@@ -425,10 +439,21 @@ Use the following format to generate the response and do not wrap it in any othe
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        feedback_history=feedback_history,
     )
 
 
-def mock_prompt(rows: int) -> str:
+def mock_prompt(rows: int, feedback: str = "", history: str = "") -> str:
+    feedback_history = ""
+    if feedback and history:
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
+Use the following feedback and history to regenerate the response:
+Feedback to improve the response:
+{feedback}
+History of previous results:
+{history}"""
+
     return f"""
 Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
 
@@ -569,6 +594,9 @@ Follow the instructions and generate the following response with no additional t
 <response>
     <sql>[raw sql query here]</sql>
 </response>
+
+{feedback_history}
+
 """
 
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/sql.py

@@ -241,7 +241,7 @@ def format_parse_error(
         message += f" found at position {adjusted_position - len(keyword)}"
     else:
         message += (
-            f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
+            f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
         )
     return message
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/sql_template_fmt.py

@@ -100,7 +100,7 @@ def _format_jinja_node(self, node: Node, max_length: int) -> bool:
                 parts = tag.code.split("\n")
                 prefix = INDENT * (node.depth[0] + node.depth[1])
                 if len(parts) > 1:
-                    tag.code = "\n".join([f'{prefix if i != 0 else ""}{part}' for i, part in enumerate(parts)])
+                    tag.code = "\n".join([f"{prefix if i != 0 else ''}{part}" for i, part in enumerate(parts)])
 
         node.value = str(tag)
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev42'
-__revision__ = '14b255c'
+__version__ = '0.0.1.dev44'
+__revision__ = '637cb4c'

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/tb/cli.py

@@ -6,7 +6,6 @@ if sys.platform == "win32":
 
 import tinybird.tb.modules.auth
 import tinybird.tb.modules.build
-import tinybird.tb.modules.build_client
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
 import tinybird.tb.modules.copy
@@ -18,6 +17,7 @@ import tinybird.tb.modules.fmt
 import tinybird.tb.modules.job
 import tinybird.tb.modules.local
 import tinybird.tb.modules.login
+import tinybird.tb.modules.materialization
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/tb/modules/build.py

@@ -12,6 +12,7 @@ import requests
 from tinybird.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
@@ -31,17 +32,17 @@ def build(ctx: click.Context, watch: bool) -> None:
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
     click.echo(FeedbackManager.highlight(message="\n» Building project..."))
-
     time_start = time.time()
 
-    def process(file_changed: Optional[str] = None) -> None:
+    def process(file_changed: Optional[str] = None, diff: Optional[str] = None) -> None:
         if file_changed and file_changed.endswith(".ndjson"):
             rebuild_fixture(project, tb_client, file_changed)
         else:
-            build_project(project, tb_client)
+            build_project(project, tb_client, file_changed)
             try:
                 if file_changed:
-                    build_and_print_resource(tb_client, file_changed)
+                    asyncio.run(folder_build(project, filenames=[file_changed]))
+                    build_and_print_resource(tb_client, file_changed, diff)
             except Exception:
                 pass
 
@@ -62,7 +63,7 @@ def build(ctx: click.Context, watch: bool) -> None:
         shell.run()
 
 
-def build_project(project: Project, tb_client: TinyB) -> None:
+def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str] = None) -> None:
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
@@ -79,6 +80,9 @@ def build_project(project: Project, tb_client: TinyB) -> None:
         project_path = project.path
         project_files = project.get_project_files()
 
+        if not project_files:
+            return
+
         for file_path in project_files:
             relative_path = str(Path(file_path).relative_to(project_path))
             fd = open(file_path, "rb")
@@ -101,17 +105,23 @@ def build_project(project: Project, tb_client: TinyB) -> None:
         if build_result == "success":
             datasources = result.get("datasources", [])
             pipes = result.get("pipes", [])
-            for ds in datasources:
-                ds_path = next((p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None)
-                if ds_path:
-                    ds_path = ds_path.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {ds_path} created"))
-            for pipe in pipes:
-                pipe_name = pipe.get("name")
-                pipe_path = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
-                if pipe_path:
-                    pipe_path = pipe_path.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {pipe_path} created"))
+            if not file_changed:
+                for ds in datasources:
+                    ds_path_str: Optional[str] = next(
+                        (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
+                    )
+                    if ds_path_str:
+                        ds_path = Path(ds_path_str)
+                        ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
+                        click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
+                for pipe in pipes:
+                    pipe_name = pipe.get("name")
+                    pipe_path_str: Optional[str] = next(
+                        (p for p in project_files if p.endswith(pipe_name + ".pipe")), None
+                    )
+                    if pipe_path_str:
+                        pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
+                        click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
 
             try:
                 for filename in project_files:
@@ -185,9 +195,17 @@ def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
         click.echo(FeedbackManager.error_exception(error=e))
 
 
-def build_and_print_resource(tb_client: TinyB, filename: str):
+def build_and_print_resource(tb_client: TinyB, filename: str, diff: Optional[str] = None):
+    table_name = diff
     resource_path = Path(filename)
-    name = resource_path.stem
-    pipeline = name if filename.endswith(".pipe") else None
-    res = asyncio.run(tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline))
-    print_table_formatted(res, name)
+    resource_name = resource_path.stem
+
+    pipeline = resource_name if filename.endswith(".pipe") else None
+
+    if not table_name:
+        table_name = resource_name
+
+    sql = f"SELECT * FROM {table_name} FORMAT JSON"
+
+    res = asyncio.run(tb_client.query(sql, pipeline=pipeline))
+    print_table_formatted(res, table_name)

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/tb/modules/cicd.py

@@ -39,7 +39,7 @@ jobs:
         working-directory: '{{ data_project_dir }}'
     services:
       tinybird:
-        image: tinybirdco/tinybird-local:latest
+        image: tinybirdco/tinybird-local:beta
         ports:
           - 80:80
     steps:
@@ -83,7 +83,7 @@ tinybird_ci_workflow:
     - tb build
     - tb test run
   services:
-    - name: tinybirdco/tinybird-local:latest
+    - name: tinybirdco/tinybird-local:beta
       alias: tinybird-local
 """
 

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/tb/modules/cli.py

@@ -6,10 +6,9 @@
 import json
 import logging
 import os
-import pprint
 from os import getcwd
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import click
 import humanfriendly
@@ -35,13 +34,7 @@ from tinybird.tb.modules.common import (
 )
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import build_graph
-from tinybird.tb.modules.datafile.common import Datafile, DatafileSyntaxError
 from tinybird.tb.modules.datafile.diff import diff_command
-from tinybird.tb.modules.datafile.exceptions import (
-    ParseException,
-)
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
@@ -66,15 +59,17 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
 @click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
-@click.option("--folder", type=str, default=os.getcwd(), help="Folder where files will be placed")
+@click.option("--folder", type=str, help="Folder where files will be placed")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro
-async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: str) -> None:
+async def cli(
+    ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: Optional[str]
+) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
-    project = Project(folder=folder)
+    project = Project(folder=folder or os.getcwd())
     # We need to unpatch for our tests not to break
     if show_tokens or not prod or ctx.invoked_subcommand == "build":
         __unpatch_click_output()
@@ -87,7 +82,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
     if debug:
         logging.basicConfig(level=logging.DEBUG)
 
-    config_temp = CLIConfig.get_project_config(project.path)
+    config_temp = CLIConfig.get_project_config(str(project.path))
     if token:
         config_temp.set_token(token)
     if host:
@@ -130,55 +125,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
     ctx.ensure_object(dict)["project"] = project
 
 
-@cli.command(hidden=True)
-@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
-@click.option("--debug", is_flag=True, default=False, help="Print internal representation")
-@click.pass_context
-def check(ctx: Context, filenames: List[str], debug: bool) -> None:
-    """Check file syntax."""
-
-    if not filenames:
-        project: Project = ctx.ensure_object(dict)["project"]
-        filenames = project.get_project_files()
-
-    def process(filenames: Iterable):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
-        incl_suffix = ".incl"
-        try:
-            for filename in filenames:
-                if os.path.isdir(filename):
-                    process(filenames=filename)
-
-                click.echo(FeedbackManager.info_processing_file(filename=filename))
-
-                file_suffix = Path(filename).suffix
-                if file_suffix == incl_suffix:
-                    click.echo(FeedbackManager.info_ignoring_incl_file(filename=filename))
-                    continue
-
-                doc: Datafile
-                parser = parser_matrix.get(file_suffix)
-                if not parser:
-                    raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
-
-                doc = parser(filename)
-
-                click.echo(FeedbackManager.success_processing_file(filename=filename))
-                if debug:
-                    pp = pprint.PrettyPrinter()
-                    for x in doc.nodes:
-                        pp.pprint(x)
-
-        except DatafileSyntaxError as e:
-            # TODO(eclbg): add the filename to the error message
-            raise CLIException(str(e))
-        except ParseException as e:
-            raise CLIException(FeedbackManager.error_exception(error=e))
-
-    process(filenames=filenames)
-
-
-@cli.command(hidden=True)
+@cli.command()
 @click.option(
     "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
 )

{tinybird-0.0.1.dev42 → tinybird-0.0.1.dev44}/tinybird/tb/modules/common.py

@@ -163,7 +163,7 @@ def generate_datafile(
     force: Optional[bool] = False,
     _format: Optional[str] = "csv",
     folder: Optional[str] = None,
-):
+) -> Path:
     p = Path(filename)
     base = Path("datasources")
     if folder:
@@ -190,6 +190,7 @@ def generate_datafile(
                     fixture_file.write(data[: data.rfind(newline)])
     else:
         click.echo(FeedbackManager.error_file_already_exists(file=f))
+    return f
 
 
 async def get_current_workspace(config: CLIConfig) -> Optional[Dict[str, Any]]: