tinybird 0.0.1.dev42__py3-none-any.whl → 0.0.1.dev44__py3-none-any.whl

tinybird/client.py

@@ -72,7 +72,7 @@ def parse_error_response(response: Response) -> str:
         if content.get("error", None):
             error = content["error"]
             if content.get("errors", None):
-                error += f' -> errors: {content.get("errors")}'
+                error += f" -> errors: {content.get('errors')}"
         else:
             error = json.dumps(response, indent=4)
         return error

tinybird/connectors.py

@@ -246,7 +246,7 @@ class BigQuery(Connector):
                     uri='{self.gcs.gs_url()}{destination}*.csv',
                     format='CSV',
                     overwrite=true,
-                    header={ "true" if with_headers else "false" },
+                    header={"true" if with_headers else "false"},
                     field_delimiter=',') AS
                     {sql}
                 """
@@ -319,7 +319,7 @@ class Snowflake(Connector):
 
     def create_stage(self):
         sql = f"""
-                    create stage "{self.options['schema']}".{self.stage()}
+                    create stage "{self.options["schema"]}".{self.stage()}
                         url='{self.gcs.gcs_url()}'
                         storage_integration = {self.storage_integration()};
                 """
@@ -337,7 +337,7 @@ class Snowflake(Connector):
                     from ({sql})
                     overwrite = true
                     file_format = (TYPE=CSV COMPRESSION=NONE ESCAPE_UNENCLOSED_FIELD=NONE FIELD_DELIMITER='|' FIELD_OPTIONALLY_ENCLOSED_BY='"' null_if=())
-                    header = {"true" if with_headers else "false" }
+                    header = {"true" if with_headers else "false"}
                     max_file_size = 2500000000;
                 """
         self.execute(sql)

tinybird/feedback_manager.py

@@ -951,7 +951,7 @@ Ready? """
     )
     success_datasource_alter = success_message("** The Data Source has been correctly updated.")
     success_datasource_kafka_connected = success_message(
-        "** Data Source '{id}' created\n" "** Kafka streaming connection configured successfully!"
+        "** Data Source '{id}' created\n** Kafka streaming connection configured successfully!"
     )
     success_datasource_shared = success_message(
         "** The Data Source {datasource} has been correctly shared with {workspace}"

tinybird/prompts.py

@@ -373,6 +373,7 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
     - The parameter within Tinybird templating syntax looks like this one {{String(my_param_name, default_value)}}.
     - If there are no parameters, you can omit parameters and generate a single test.
     - The format of the parameters is the following: ?param1=value1&param2=value2&param3=value3
+    - If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive.
 </instructions>
 
 This is an example of a test with parameters:
@@ -396,9 +397,19 @@ Follow the instructions and generate the following response with no additional t
 """
 
 
-def create_prompt(existing_resources: str) -> str:
+def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
+    feedback_history = ""
+    if feedback and history:
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
+Use the following feedback and history to regenerate the response:
+Feedback to improve the response:
+{feedback}
+History of previous results:
+{history}"""
+
     return """
-You are a Tinybird expert. You will be given a prompt to generate Tinybird resources: datasources and/or pipes.
+You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
 <existing_resources>{existing_resources}</existing_resources>
 {datasource_instructions}
 {pipe_instructions}
@@ -407,6 +418,9 @@ You are a Tinybird expert. You will be given a prompt to generate Tinybird resou
 {pipe_example}
 {copy_pipe_instructions}
 {materialized_pipe_instructions}
+
+{feedback_history}
+
 Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
 <response>
     <resource>
@@ -425,10 +439,21 @@ Use the following format to generate the response and do not wrap it in any othe
         pipe_example=pipe_example,
         copy_pipe_instructions=copy_pipe_instructions,
         materialized_pipe_instructions=materialized_pipe_instructions,
+        feedback_history=feedback_history,
     )
 
 
-def mock_prompt(rows: int) -> str:
+def mock_prompt(rows: int, feedback: str = "", history: str = "") -> str:
+    feedback_history = ""
+    if feedback and history:
+        feedback_history = f"""In case the <feedback> and <history> tags are present and not empty, 
+it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
+Use the following feedback and history to regenerate the response:
+Feedback to improve the response:
+{feedback}
+History of previous results:
+{history}"""
+
     return f"""
 Given the schema for a Tinybird datasource, return a can you create a clickhouse sql query to generate some random data that matches that schema.
 
@@ -569,6 +594,9 @@ Follow the instructions and generate the following response with no additional t
 <response>
     <sql>[raw sql query here]</sql>
 </response>
+
+{feedback_history}
+
 """
 
 

tinybird/sql.py

@@ -241,7 +241,7 @@ def format_parse_error(
         message += f" found at position {adjusted_position - len(keyword)}"
     else:
         message += (
-            f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
+            f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
         )
     return message
 

tinybird/sql_template_fmt.py

@@ -100,7 +100,7 @@ def _format_jinja_node(self, node: Node, max_length: int) -> bool:
                 parts = tag.code.split("\n")
                 prefix = INDENT * (node.depth[0] + node.depth[1])
                 if len(parts) > 1:
-                    tag.code = "\n".join([f'{prefix if i != 0 else ""}{part}' for i, part in enumerate(parts)])
+                    tag.code = "\n".join([f"{prefix if i != 0 else ''}{part}" for i, part in enumerate(parts)])
 
         node.value = str(tag)
 

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev42'
-__revision__ = '14b255c'
+__version__ = '0.0.1.dev44'
+__revision__ = '637cb4c'

tinybird/tb/cli.py

@@ -6,7 +6,6 @@ if sys.platform == "win32":
 
 import tinybird.tb.modules.auth
 import tinybird.tb.modules.build
-import tinybird.tb.modules.build_client
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
 import tinybird.tb.modules.copy
@@ -18,6 +17,7 @@ import tinybird.tb.modules.fmt
 import tinybird.tb.modules.job
 import tinybird.tb.modules.local
 import tinybird.tb.modules.login
+import tinybird.tb.modules.materialization
 import tinybird.tb.modules.mock
 import tinybird.tb.modules.pipe
 import tinybird.tb.modules.tag

tinybird/tb/modules/build.py

@@ -12,6 +12,7 @@ import requests
 from tinybird.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.datafile.build import folder_build
 from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
@@ -31,17 +32,17 @@ def build(ctx: click.Context, watch: bool) -> None:
     project: Project = ctx.ensure_object(dict)["project"]
     tb_client = asyncio.run(get_tinybird_local_client(str(project.path)))
     click.echo(FeedbackManager.highlight(message="\n» Building project..."))
-
     time_start = time.time()
 
-    def process(file_changed: Optional[str] = None) -> None:
+    def process(file_changed: Optional[str] = None, diff: Optional[str] = None) -> None:
         if file_changed and file_changed.endswith(".ndjson"):
             rebuild_fixture(project, tb_client, file_changed)
         else:
-            build_project(project, tb_client)
+            build_project(project, tb_client, file_changed)
             try:
                 if file_changed:
-                    build_and_print_resource(tb_client, file_changed)
+                    asyncio.run(folder_build(project, filenames=[file_changed]))
+                    build_and_print_resource(tb_client, file_changed, diff)
             except Exception:
                 pass
 
@@ -62,7 +63,7 @@ def build(ctx: click.Context, watch: bool) -> None:
         shell.run()
 
 
-def build_project(project: Project, tb_client: TinyB) -> None:
+def build_project(project: Project, tb_client: TinyB, file_changed: Optional[str] = None) -> None:
     MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
     DATAFILE_TYPE_TO_CONTENT_TYPE = {
         ".datasource": "text/plain",
@@ -79,6 +80,9 @@ def build_project(project: Project, tb_client: TinyB) -> None:
         project_path = project.path
         project_files = project.get_project_files()
 
+        if not project_files:
+            return
+
         for file_path in project_files:
             relative_path = str(Path(file_path).relative_to(project_path))
             fd = open(file_path, "rb")
@@ -101,17 +105,23 @@ def build_project(project: Project, tb_client: TinyB) -> None:
         if build_result == "success":
             datasources = result.get("datasources", [])
             pipes = result.get("pipes", [])
-            for ds in datasources:
-                ds_path = next((p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None)
-                if ds_path:
-                    ds_path = ds_path.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {ds_path} created"))
-            for pipe in pipes:
-                pipe_name = pipe.get("name")
-                pipe_path = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
-                if pipe_path:
-                    pipe_path = pipe_path.replace(f"{project.folder}/", "")
-                    click.echo(FeedbackManager.info(message=f"✓ {pipe_path} created"))
+            if not file_changed:
+                for ds in datasources:
+                    ds_path_str: Optional[str] = next(
+                        (p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None
+                    )
+                    if ds_path_str:
+                        ds_path = Path(ds_path_str)
+                        ds_path_str = ds_path_str.replace(f"{project.folder}/", "")
+                        click.echo(FeedbackManager.info(message=f"✓ {ds_path_str} created"))
+                for pipe in pipes:
+                    pipe_name = pipe.get("name")
+                    pipe_path_str: Optional[str] = next(
+                        (p for p in project_files if p.endswith(pipe_name + ".pipe")), None
+                    )
+                    if pipe_path_str:
+                        pipe_path_str = pipe_path_str.replace(f"{project.folder}/", "")
+                        click.echo(FeedbackManager.info(message=f"✓ {pipe_path_str} created"))
 
             try:
                 for filename in project_files:
@@ -185,9 +195,17 @@ def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
         click.echo(FeedbackManager.error_exception(error=e))
 
 
-def build_and_print_resource(tb_client: TinyB, filename: str):
+def build_and_print_resource(tb_client: TinyB, filename: str, diff: Optional[str] = None):
+    table_name = diff
     resource_path = Path(filename)
-    name = resource_path.stem
-    pipeline = name if filename.endswith(".pipe") else None
-    res = asyncio.run(tb_client.query(f"SELECT * FROM {name} FORMAT JSON", pipeline=pipeline))
-    print_table_formatted(res, name)
+    resource_name = resource_path.stem
+
+    pipeline = resource_name if filename.endswith(".pipe") else None
+
+    if not table_name:
+        table_name = resource_name
+
+    sql = f"SELECT * FROM {table_name} FORMAT JSON"
+
+    res = asyncio.run(tb_client.query(sql, pipeline=pipeline))
+    print_table_formatted(res, table_name)

tinybird/tb/modules/cicd.py

@@ -39,7 +39,7 @@ jobs:
         working-directory: '{{ data_project_dir }}'
     services:
       tinybird:
-        image: tinybirdco/tinybird-local:latest
+        image: tinybirdco/tinybird-local:beta
         ports:
           - 80:80
     steps:
@@ -83,7 +83,7 @@ tinybird_ci_workflow:
     - tb build
     - tb test run
   services:
-    - name: tinybirdco/tinybird-local:latest
+    - name: tinybirdco/tinybird-local:beta
       alias: tinybird-local
 """
 

tinybird/tb/modules/cli.py

@@ -6,10 +6,9 @@
 import json
 import logging
 import os
-import pprint
 from os import getcwd
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import click
 import humanfriendly
@@ -35,13 +34,7 @@ from tinybird.tb.modules.common import (
 )
 from tinybird.tb.modules.config import CLIConfig
 from tinybird.tb.modules.datafile.build import build_graph
-from tinybird.tb.modules.datafile.common import Datafile, DatafileSyntaxError
 from tinybird.tb.modules.datafile.diff import diff_command
-from tinybird.tb.modules.datafile.exceptions import (
-    ParseException,
-)
-from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
-from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.datafile.pull import folder_pull
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.llm import LLM
@@ -66,15 +59,17 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
 @click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
-@click.option("--folder", type=str, default=os.getcwd(), help="Folder where files will be placed")
+@click.option("--folder", type=str, help="Folder where files will be placed")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro
-async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: str) -> None:
+async def cli(
+    ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool, folder: Optional[str]
+) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
-    project = Project(folder=folder)
+    project = Project(folder=folder or os.getcwd())
     # We need to unpatch for our tests not to break
     if show_tokens or not prod or ctx.invoked_subcommand == "build":
         __unpatch_click_output()
@@ -87,7 +82,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
     if debug:
         logging.basicConfig(level=logging.DEBUG)
 
-    config_temp = CLIConfig.get_project_config(project.path)
+    config_temp = CLIConfig.get_project_config(str(project.path))
     if token:
         config_temp.set_token(token)
     if host:
@@ -130,55 +125,7 @@ async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: boo
     ctx.ensure_object(dict)["project"] = project
 
 
-@cli.command(hidden=True)
-@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
-@click.option("--debug", is_flag=True, default=False, help="Print internal representation")
-@click.pass_context
-def check(ctx: Context, filenames: List[str], debug: bool) -> None:
-    """Check file syntax."""
-
-    if not filenames:
-        project: Project = ctx.ensure_object(dict)["project"]
-        filenames = project.get_project_files()
-
-    def process(filenames: Iterable):
-        parser_matrix = {".pipe": parse_pipe, ".datasource": parse_datasource}
-        incl_suffix = ".incl"
-        try:
-            for filename in filenames:
-                if os.path.isdir(filename):
-                    process(filenames=filename)
-
-                click.echo(FeedbackManager.info_processing_file(filename=filename))
-
-                file_suffix = Path(filename).suffix
-                if file_suffix == incl_suffix:
-                    click.echo(FeedbackManager.info_ignoring_incl_file(filename=filename))
-                    continue
-
-                doc: Datafile
-                parser = parser_matrix.get(file_suffix)
-                if not parser:
-                    raise ParseException(FeedbackManager.error_unsupported_datafile(extension=file_suffix))
-
-                doc = parser(filename)
-
-                click.echo(FeedbackManager.success_processing_file(filename=filename))
-                if debug:
-                    pp = pprint.PrettyPrinter()
-                    for x in doc.nodes:
-                        pp.pprint(x)
-
-        except DatafileSyntaxError as e:
-            # TODO(eclbg): add the filename to the error message
-            raise CLIException(str(e))
-        except ParseException as e:
-            raise CLIException(FeedbackManager.error_exception(error=e))
-
-    process(filenames=filenames)
-
-
-@cli.command(hidden=True)
+@cli.command()
 @click.option(
     "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
 )

tinybird/tb/modules/common.py

@@ -163,7 +163,7 @@ def generate_datafile(
     force: Optional[bool] = False,
     _format: Optional[str] = "csv",
     folder: Optional[str] = None,
-):
+) -> Path:
     p = Path(filename)
     base = Path("datasources")
     if folder:
@@ -190,6 +190,7 @@ def generate_datafile(
                     fixture_file.write(data[: data.rfind(newline)])
     else:
         click.echo(FeedbackManager.error_file_already_exists(file=f))
+    return f
 
 
 async def get_current_workspace(config: CLIConfig) -> Optional[Dict[str, Any]]:

tinybird/tb/modules/copy.py

@@ -5,19 +5,20 @@
 
 import json
 import re
+from typing import Optional, Tuple
 
 import click
 from click import Context
 
-from tinybird.client import TinyB
+from tinybird.client import AuthNoTokenException, TinyB
 from tinybird.tb.modules.cli import cli
-from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table
+from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table, wait_job
 from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.exceptions import CLIPipeException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 
 
-@cli.group(hidden=True)
+@cli.group()
 @click.pass_context
 def copy(ctx):
     """Copy pipe commands"""
@@ -38,10 +39,10 @@ async def copy_ls(ctx: Context, match: str, format_: str):
     """List copy pipes"""
 
     client: TinyB = ctx.ensure_object(dict)["client"]
-    pipes = await client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at")
+    pipes = await client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
     copies = [p for p in pipes if p.get("type") == "copy"]
     copies = sorted(copies, key=lambda p: p["updated_at"])
-    columns = ["name", "updated at", "nodes", "url"]
+    columns = ["name", "updated at", "nodes"]
     table_human_readable = []
     table_machine_readable = []
     pattern = re.compile(match) if match else None
@@ -66,3 +67,93 @@ async def copy_ls(ctx: Context, match: str, format_: str):
         click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
     else:
         raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
+
+
+@copy.command(name="run", short_help="Run an on-demand copy job")
+@click.argument("pipe_name_or_id")
+@click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
+@click.option(
+    "--mode", type=click.Choice(["append", "replace"], case_sensitive=True), default=None, help="Copy strategy"
+)
+@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
+@click.option(
+    "--param",
+    nargs=1,
+    type=str,
+    multiple=True,
+    default=None,
+    help="Key and value of the params you want the Copy pipe to be called with. For example: tb pipe copy run <my_copy_pipe> --param foo=bar",
+)
+@click.pass_context
+@coro
+async def copy_run(
+    ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, yes: bool, param: Optional[Tuple[str]]
+):
+    """Run an on-demand copy job"""
+
+    params = dict(key_value.split("=") for key_value in param) if param else {}
+
+    if yes or click.confirm(FeedbackManager.warning_confirm_copy_pipe(pipe=pipe_name_or_id)):
+        click.echo(FeedbackManager.info_copy_job_running(pipe=pipe_name_or_id))
+        client: TinyB = ctx.ensure_object(dict)["client"]
+
+        try:
+            response = await client.pipe_run_copy(pipe_name_or_id, params, mode)
+
+            job_id = response["job"]["job_id"]
+            job_url = response["job"]["job_url"]
+            target_datasource_id = response["tags"]["copy_target_datasource"]
+            target_datasource = await client.get_datasource(target_datasource_id)
+            target_datasource_name = target_datasource["name"]
+            click.echo(
+                FeedbackManager.success_copy_job_created(target_datasource=target_datasource_name, job_url=job_url)
+            )
+
+            if wait:
+                await wait_job(client, job_id, job_url, "** Copying data")
+                click.echo(FeedbackManager.success_data_copied_to_ds(target_datasource=target_datasource_name))
+
+        except AuthNoTokenException:
+            raise
+        except Exception as e:
+            raise CLIPipeException(FeedbackManager.error_creating_copy_job(error=e))
+
+
+@copy.command(name="resume", short_help="Resume a paused copy pipe")
+@click.argument("pipe_name_or_id")
+@click.pass_context
+@coro
+async def copy_resume(ctx: click.Context, pipe_name_or_id: str):
+    """Resume a paused copy pipe"""
+
+    click.echo(FeedbackManager.info_copy_pipe_resuming(pipe=pipe_name_or_id))
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        await client.pipe_resume_copy(pipe_name_or_id)
+        click.echo(FeedbackManager.success_copy_pipe_resumed(pipe=pipe_name_or_id))
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_resuming_copy_pipe(error=e))
+
+
+@copy.command(name="pause", short_help="Pause a running copy pipe")
+@click.argument("pipe_name_or_id")
+@click.pass_context
+@coro
+async def copy_pause(ctx: click.Context, pipe_name_or_id: str):
+    """Pause a running copy pipe"""
+
+    click.echo(FeedbackManager.info_copy_pipe_pausing(pipe=pipe_name_or_id))
+    client: TinyB = ctx.ensure_object(dict)["client"]
+
+    try:
+        await client.pipe_pause_copy(pipe_name_or_id)
+        click.echo(FeedbackManager.success_copy_pipe_paused(pipe=pipe_name_or_id))
+
+    except AuthNoTokenException:
+        raise
+    except Exception as e:
+        raise CLIPipeException(FeedbackManager.error_pausing_copy_pipe(error=e))