tinybird 0.0.1.dev190__py3-none-any.whl → 0.0.1.dev191__py3-none-any.whl

tinybird/prompts.py

@@ -1010,3 +1010,22 @@ Current README.md file:
 <readme>[readme content here]</readme>
 </readme_instructions>
 """
+
+
+def quarantine_prompt(datasource_definition: str) -> str:
+    return f"""
+- You are an expert in Tinybird.
+- You are given a list of rows that went to quarantine during ingestion because of data quality issues.
+- Return the errors in a human readable format so the user can understand what is the problem and fix it.
+- Be concise and to the point.
+- Do not mention clickhouse tables. Refer to data sources instead.
+- The possible fixes recommended fixes are:
+    - Changing a column type in the datasource definition: tell the user what to change in the datasource file, then build again before appending the data.
+    - Changing the data because of the wrong data type: tell the user what to change in the data file, then append the data again.
+- The format of the response always inside the tag <quarantine_errors>[response]</quarantine_errors>
+- Do not use markdown format in the response, because it is a CLI output.
+- The datasource definition is the following:
+<datasource_definition>
+{datasource_definition}
+</datasource_definition>
+"""

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev190'
-__revision__ = '8a00bcf'
+__version__ = '0.0.1.dev191'
+__revision__ = 'c8535c7'

tinybird/tb/modules/cli.py

@@ -28,7 +28,9 @@ from tinybird.tb.modules.common import (
     CLIException,
     _get_tb_client,
     coro,
+    echo_json,
     echo_safe_format_table,
+    force_echo,
     getenv_bool,
     try_update_config_with_remote,
 )
@@ -72,7 +74,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
 @click.option(
-    "--output", type=click.Choice(["json", "human"], case_sensitive=False), default="human", help="Output format"
+    "--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
 )
 @click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
 @click.version_option(version=VERSION)
@@ -95,7 +97,7 @@ async def cli(
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
     # We need to unpatch for our tests not to break
-    if output == "json":
+    if output != "human":
         __hide_click_output()
     else:
         if show_tokens or not cloud or ctx.invoked_subcommand == "build":
@@ -195,17 +197,10 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
 
 @cli.command()
 @click.argument("query", required=False)
-@click.option("--rows_limit", default=100, help="Max number of rows retrieved")
+@click.option("--rows-limit", default=100, help="Max number of rows retrieved")
 @click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
 @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
 @click.option("--node", default=None, help="The NODE name")
-@click.option(
-    "--format",
-    "format_",
-    type=click.Choice(["json", "csv", "human"], case_sensitive=False),
-    default="human",
-    help="Output format",
-)
 @click.option("--stats/--no-stats", default=False, help="Show query stats")
 @click.pass_context
 @coro
@@ -216,13 +211,13 @@ async def sql(
     pipeline: Optional[str],
     pipe: Optional[str],
     node: Optional[str],
-    format_: str,
     stats: bool,
 ) -> None:
     """Run SQL query over data sources and pipes."""
-
     client = ctx.ensure_object(dict)["client"]
-    req_format = "CSVWithNames" if format_ == "csv" else "JSON"
+    output = ctx.ensure_object(dict)["output"]
+
+    req_format = "CSVWithNames" if output == "csv" else "JSON"
     res = None
     try:
         if query:
@@ -275,11 +270,11 @@ async def sql(
         bytes_read = humanfriendly.format_size(stats_dict["bytes_read"])
         click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
 
-    if format_ == "csv":
-        click.echo(res)
+    if output == "csv":
+        force_echo(str(res))
     elif isinstance(res, dict) and "data" in res and res["data"]:
-        if format_ == "json":
-            click.echo(json.dumps(res, indent=8))
+        if output == "json":
+            echo_json(res, indent=8)
         else:
             dd = []
             for d in res["data"]:

tinybird/tb/modules/common.py

@@ -2250,5 +2250,9 @@ def get_error_event(error: str) -> Tuple[str, str]:
     return error_event, silent_error_msg
 
 
-def echo_json(data: Dict[str, Any]) -> None:
-    click.echo(json.dumps(data), force_output=True)  # type: ignore
+def force_echo(string: str) -> None:
+    click.echo(string, force_output=True)  # type: ignore
+
+
+def echo_json(data: Dict[str, Any], indent: Union[None, int, str] = None) -> None:
+    force_echo(json.dumps(data, indent=indent))

tinybird/tb/modules/datasource.py

@@ -18,6 +18,7 @@ import humanfriendly
 import requests
 from click import Context
 
+from tinybird.prompts import quarantine_prompt
 from tinybird.syncasync import sync_to_async
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.tb.modules.cli import cli
@@ -42,7 +43,10 @@ from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.llm import LLM
+from tinybird.tb.modules.llm_utils import extract_xml
 from tinybird.tb.modules.project import Project
+from tinybird.tb.modules.telemetry import add_telemetry_event
 
 
 @cli.group()
@@ -151,6 +155,7 @@ async def datasource_append(
     """
     env: str = ctx.ensure_object(dict)["env"]
     client: TinyB = ctx.obj["client"]
+    project: Project = ctx.ensure_object(dict)["project"]
 
     # If data is passed as argument, we detect if it's a JSON object, a URL or a file
     if data:
@@ -244,15 +249,10 @@ async def datasource_append(
 
     if events:
         click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
-        try:
-            json_data = json.loads(events)
-        except Exception:
-            raise CLIDatasourceException(FeedbackManager.error(message="Invalid events data"))
-
         response = await sync_to_async(requests.post)(
             f"{client.host}/v0/events?name={datasource_name}",
             headers={"Authorization": f"Bearer {client.token}"},
-            json=json_data,
+            data=events,
         )
 
         try:
@@ -269,21 +269,32 @@ async def datasource_append(
                 )
             )
         if quarantined_rows > 0:
-            raise CLIDatasourceException(
+            click.echo(
                 FeedbackManager.error(
-                    message=f"{quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
+                    message=f"✗ {quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
                 )
             )
+            await analyze_quarantine(datasource_name, project, client)
+            return
     else:
         click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
-        await push_data(
-            client,
-            datasource_name,
-            data,
-            mode="append",
-            concurrency=concurrency,
-            silent=True,
-        )
+        try:
+            await push_data(
+                client,
+                datasource_name,
+                data,
+                mode="append",
+                concurrency=concurrency,
+                silent=True,
+            )
+        except Exception as e:
+            is_quarantined = "quarantine" in str(e)
+            click.echo(FeedbackManager.error(message="✗ " + str(e)))
+            if is_quarantined:
+                await analyze_quarantine(datasource_name, project, client)
+                return
+            else:
+                raise e
         click.echo(FeedbackManager.success(message="✓ Rows appended!"))
 
 
@@ -902,3 +913,33 @@ def generate_short_id():
 
 def generate_kafka_group_id(topic: str):
     return f"{topic}_{int(datetime.timestamp(datetime.now()))}"
+
+
+async def analyze_quarantine(datasource_name: str, project: Project, client: TinyB):
+    config = CLIConfig.get_project_config()
+    res = await client.query(
+        f"SELECT * FROM {datasource_name}_quarantine ORDER BY insertion_date DESC LIMIT 1 FORMAT JSON"
+    )
+    quarantine_data = res["data"]
+    error_message = json.dumps(res["data"])
+    user_token = config.get_user_token()
+    click.echo(FeedbackManager.gray(message=f"\n» Analyzing errors in {datasource_name}_quarantine..."))
+    if user_token:
+        llm = LLM(user_token=user_token, host=config.get_client().host)
+        ds_filenames = project.get_datasource_files()
+        datasource_definition = next(
+            (Path(f).read_text() for f in ds_filenames if f.endswith(f"{datasource_name}.datasource")), ""
+        )
+        response_llm = llm.ask(
+            system_prompt=quarantine_prompt(datasource_definition),
+            prompt=f"The quarantine errors are:\n{json.dumps(quarantine_data)}",
+        )
+        response = extract_xml(response_llm, "quarantine_errors")
+        error_message += "\n" + response
+        click.echo(response)
+    else:
+        echo_safe_humanfriendly_tables_format_smart_table(
+            data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
+        )
+
+    add_telemetry_event("datasource_error", error=f"quarantine_error: {error_message}")

tinybird/tb/modules/feedback_manager.py

@@ -437,6 +437,9 @@ class FeedbackManager:
     error_tag_not_found = error_message("Tag {tag_name} not found.")
     error_build_failed = error_message("Build failed")
     error_request_failed = error_message("Request failed with status code {status_code}, please try again later.")
+    error_invalid_output_format = error_message(
+        "Invalid output format for this command. Supported formats are: {formats}"
+    )
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(

tinybird/tb/modules/info.py

@@ -6,7 +6,7 @@ import click
 from tinybird.tb.client import TinyB
 from tinybird.tb.config import get_display_cloud_host
 from tinybird.tb.modules.cli import CLIConfig, cli
-from tinybird.tb.modules.common import coro, echo_json, format_robust_table
+from tinybird.tb.modules.common import coro, echo_json, force_echo, format_robust_table
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_tinybird_local_config
 from tinybird.tb.modules.project import Project
@@ -19,14 +19,19 @@ async def info(ctx: click.Context) -> None:
     """Get information about the project that is currently being used"""
     ctx_config = ctx.ensure_object(dict)["config"]
     project: Project = ctx.ensure_object(dict)["project"]
-    is_json = ctx.ensure_object(dict)["output"] == "json"
+    output = ctx.ensure_object(dict)["output"]
+
+    if output not in {"human", "json"}:
+        force_echo(FeedbackManager.error_invalid_output_format(formats=", ".join(["human", "json"])))
+        return
+
     click.echo(FeedbackManager.highlight(message="» Tinybird Cloud:"))
     cloud_table, cloud_columns = await get_cloud_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
     local_table, local_columns = await get_local_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Project:"))
     project_table, project_columns = await get_project_info(project.folder)
-    if is_json:
+    if output == "json":
         cloud_data = {}
         if cloud_columns and cloud_table and isinstance(cloud_table, list) and len(cloud_table) > 0:
             cloud_data = {column: cloud_table[0][i] for i, column in enumerate(cloud_columns)}

{tinybird-0.0.1.dev190.dist-info → tinybird-0.0.1.dev191.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev190
+Version: 0.0.1.dev191
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev190.dist-info → tinybird-0.0.1.dev191.dist-info}/RECORD

@@ -3,7 +3,7 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/prompts.py,sha256=uFzlVsGgWFwEOBhXkaLL4zY-wz8jCQgTFiMEfoFcgro,36550
+tinybird/prompts.py,sha256=rpgvMpR103niDnoDMSz8hRAYBdgfrorfD5-7g23BMQQ,37596
 tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
 tinybird/sql_template.py,sha256=hWW8JawSWLl9GeWPYkC_Yrxj7P0MHEVMJ0Px9bedEgM,99817
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
@@ -12,27 +12,27 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
-tinybird/tb/__cli__.py,sha256=Uk4h64toVqAfnRJ4rIo4K3vfl_FunHW228-sIHLBDt0,247
+tinybird/tb/__cli__.py,sha256=IuO2njWcdIXnuscWmXWEMTbCM9GnrTyYky59OtsEi3I,247
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
 tinybird/tb/client.py,sha256=CO-dQw8h28X6T6IO-Z79yPBKaJQT1Rwya5b6gexvw58,56491
 tinybird/tb/config.py,sha256=jT9xndpeCY_g0HdB5qE2EquC0TFRRnkPnQFWZWd04jo,3998
 tinybird/tb/modules/build.py,sha256=KvF0s8hGgY_rZs7jSqYiauCk3MAlCmW_gQtnsJDJWBk,19411
 tinybird/tb/modules/cicd.py,sha256=Njb6eZOHHbUkoJJx6KoixO9PsfA_T-3Ybkya9-50Ca8,7328
-tinybird/tb/modules/cli.py,sha256=dXZs-MuqYPvxStVj7aLg36LwXtEB8NzTobDmHV9nzZI,15508
-tinybird/tb/modules/common.py,sha256=6SYrDH-GhWm8N8S2GmHdIbvf1mUYIzWYlz0WcjDIhGw,83947
+tinybird/tb/modules/cli.py,sha256=LW77oBqvZ6QB5X96158Kp13Tl24vKdxlyWvsSawpmCo,15399
+tinybird/tb/modules/common.py,sha256=2NRDRll0czmYjwLh3qv3DYL9aP8XgRkRAv5S3meCGfM,84062
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
 tinybird/tb/modules/connection.py,sha256=z1xWP2gtjKEbjc4ZF1aD7QUgl8V--wf2IRNy-4sRFm8,9779
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
 tinybird/tb/modules/create.py,sha256=2uW-4t7c7e4xkZ-GpK_8XaA-nuXwklq7rTks4k6qrtI,20917
-tinybird/tb/modules/datasource.py,sha256=gycHinGTDxZMDqwbIaRJkR2SUxdhbr1prCFQHkssgVs,35124
+tinybird/tb/modules/datasource.py,sha256=Uomj3bpVL5m4A0_ezWFXvQAjVER4EfmDz5gFkfGR7eo,37027
 tinybird/tb/modules/deployment.py,sha256=OGJdriqywhCtsG7rKLFtVSLjoEbbva1Nb29-jQBk3wM,27432
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=XySDt3pk66vxOZ0egUfz4bY8bEk3BjOXkv-L0OIJ3sc,12083
 tinybird/tb/modules/exceptions.py,sha256=5jK91w1LPmtqIUfDpHe_Op5OxGz8-p1BPgtLREMIni0,5217
-tinybird/tb/modules/feedback_manager.py,sha256=sWJYBIZDPYLFWMKPv7FoVLbQxE7xx85bn5SbK9yYXoE,76782
-tinybird/tb/modules/info.py,sha256=iKeFbFkos7vYaBU7Vr5SI-fa1x7AbuUHB748jsGsaA4,5944
+tinybird/tb/modules/feedback_manager.py,sha256=cnMGTCXa04Zmwq0tPHuAeq_hxqTz3F8c7mXqTJ-MZsw,76920
+tinybird/tb/modules/info.py,sha256=qsXr2g45jRv2xD4Kt2sgsT0AR8sjwrAZf2s7mI-zxL4,6113
 tinybird/tb/modules/infra.py,sha256=fve30Gj3mG9zbquGxS2e4ipcOYOxviWQCpNFfEzJN_Q,33195
 tinybird/tb/modules/job.py,sha256=AsUCRNzy7HG5oJ4fyk9NpIm5NtNJgBZSy8MtJdXBe5A,3167
 tinybird/tb/modules/llm.py,sha256=KfsCYmKeW1VQz0iDZhGKCRkQv_Y3kTHh6JuxvofOguE,1076
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev190.dist-info/METADATA,sha256=MDRLrgiaBGisoaGthlSBKhjMSvdw8XSWPF5VEUsjrvM,1608
-tinybird-0.0.1.dev190.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev190.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev190.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev190.dist-info/RECORD,,
+tinybird-0.0.1.dev191.dist-info/METADATA,sha256=6DYh0XzR-JKDXhw_nDCp0ZXErl2PTGkPEBMDqKLTLB8,1608
+tinybird-0.0.1.dev191.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev191.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev191.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev191.dist-info/RECORD,,