tinybird 0.0.1.dev190__tar.gz → 0.0.1.dev191__tar.gz

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev190
+Version: 0.0.1.dev191
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/prompts.py

@@ -1010,3 +1010,22 @@ Current README.md file:
 <readme>[readme content here]</readme>
 </readme_instructions>
 """
+
+
+def quarantine_prompt(datasource_definition: str) -> str:
+    return f"""
+- You are an expert in Tinybird.
+- You are given a list of rows that went to quarantine during ingestion because of data quality issues.
+- Return the errors in a human readable format so the user can understand what is the problem and fix it.
+- Be concise and to the point.
+- Do not mention clickhouse tables. Refer to data sources instead.
+- The possible fixes recommended fixes are:
+    - Changing a column type in the datasource definition: tell the user what to change in the datasource file, then build again before appending the data.
+    - Changing the data because of the wrong data type: tell the user what to change in the data file, then append the data again.
+- The format of the response always inside the tag <quarantine_errors>[response]</quarantine_errors>
+- Do not use markdown format in the response, because it is a CLI output.
+- The datasource definition is the following:
+<datasource_definition>
+{datasource_definition}
+</datasource_definition>
+"""

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev190'
-__revision__ = '8a00bcf'
+__version__ = '0.0.1.dev191'
+__revision__ = 'c8535c7'

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/tb/modules/cli.py

@@ -28,7 +28,9 @@ from tinybird.tb.modules.common import (
     CLIException,
     _get_tb_client,
     coro,
+    echo_json,
     echo_safe_format_table,
+    force_echo,
     getenv_bool,
     try_update_config_with_remote,
 )
@@ -72,7 +74,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
 @click.option(
-    "--output", type=click.Choice(["json", "human"], case_sensitive=False), default="human", help="Output format"
+    "--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
 )
 @click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
 @click.version_option(version=VERSION)
@@ -95,7 +97,7 @@ async def cli(
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
     # We need to unpatch for our tests not to break
-    if output == "json":
+    if output != "human":
         __hide_click_output()
     else:
         if show_tokens or not cloud or ctx.invoked_subcommand == "build":
@@ -195,17 +197,10 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
 
 @cli.command()
 @click.argument("query", required=False)
-@click.option("--rows_limit", default=100, help="Max number of rows retrieved")
+@click.option("--rows-limit", default=100, help="Max number of rows retrieved")
 @click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
 @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
 @click.option("--node", default=None, help="The NODE name")
-@click.option(
-    "--format",
-    "format_",
-    type=click.Choice(["json", "csv", "human"], case_sensitive=False),
-    default="human",
-    help="Output format",
-)
 @click.option("--stats/--no-stats", default=False, help="Show query stats")
 @click.pass_context
 @coro
@@ -216,13 +211,13 @@ async def sql(
     pipeline: Optional[str],
     pipe: Optional[str],
     node: Optional[str],
-    format_: str,
     stats: bool,
 ) -> None:
     """Run SQL query over data sources and pipes."""
-
     client = ctx.ensure_object(dict)["client"]
-    req_format = "CSVWithNames" if format_ == "csv" else "JSON"
+    output = ctx.ensure_object(dict)["output"]
+
+    req_format = "CSVWithNames" if output == "csv" else "JSON"
     res = None
     try:
         if query:
@@ -275,11 +270,11 @@ async def sql(
         bytes_read = humanfriendly.format_size(stats_dict["bytes_read"])
         click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
 
-    if format_ == "csv":
-        click.echo(res)
+    if output == "csv":
+        force_echo(str(res))
     elif isinstance(res, dict) and "data" in res and res["data"]:
-        if format_ == "json":
-            click.echo(json.dumps(res, indent=8))
+        if output == "json":
+            echo_json(res, indent=8)
         else:
             dd = []
             for d in res["data"]:

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/tb/modules/common.py

@@ -2250,5 +2250,9 @@ def get_error_event(error: str) -> Tuple[str, str]:
     return error_event, silent_error_msg
 
 
-def echo_json(data: Dict[str, Any]) -> None:
-    click.echo(json.dumps(data), force_output=True)  # type: ignore
+def force_echo(string: str) -> None:
+    click.echo(string, force_output=True)  # type: ignore
+
+
+def echo_json(data: Dict[str, Any], indent: Union[None, int, str] = None) -> None:
+    force_echo(json.dumps(data, indent=indent))

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/tb/modules/datasource.py

@@ -18,6 +18,7 @@ import humanfriendly
 import requests
 from click import Context
 
+from tinybird.prompts import quarantine_prompt
 from tinybird.syncasync import sync_to_async
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.tb.modules.cli import cli
@@ -42,7 +43,10 @@ from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.llm import LLM
+from tinybird.tb.modules.llm_utils import extract_xml
 from tinybird.tb.modules.project import Project
+from tinybird.tb.modules.telemetry import add_telemetry_event
 
 
 @cli.group()
@@ -151,6 +155,7 @@ async def datasource_append(
     """
     env: str = ctx.ensure_object(dict)["env"]
     client: TinyB = ctx.obj["client"]
+    project: Project = ctx.ensure_object(dict)["project"]
 
     # If data is passed as argument, we detect if it's a JSON object, a URL or a file
     if data:
@@ -244,15 +249,10 @@ async def datasource_append(
 
     if events:
         click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
-        try:
-            json_data = json.loads(events)
-        except Exception:
-            raise CLIDatasourceException(FeedbackManager.error(message="Invalid events data"))
-
         response = await sync_to_async(requests.post)(
             f"{client.host}/v0/events?name={datasource_name}",
             headers={"Authorization": f"Bearer {client.token}"},
-            json=json_data,
+            data=events,
         )
 
         try:
@@ -269,21 +269,32 @@ async def datasource_append(
                 )
             )
         if quarantined_rows > 0:
-            raise CLIDatasourceException(
+            click.echo(
                 FeedbackManager.error(
-                    message=f"{quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
+                    message=f"✗ {quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
                 )
             )
+            await analyze_quarantine(datasource_name, project, client)
+            return
     else:
         click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
-        await push_data(
-            client,
-            datasource_name,
-            data,
-            mode="append",
-            concurrency=concurrency,
-            silent=True,
-        )
+        try:
+            await push_data(
+                client,
+                datasource_name,
+                data,
+                mode="append",
+                concurrency=concurrency,
+                silent=True,
+            )
+        except Exception as e:
+            is_quarantined = "quarantine" in str(e)
+            click.echo(FeedbackManager.error(message="✗ " + str(e)))
+            if is_quarantined:
+                await analyze_quarantine(datasource_name, project, client)
+                return
+            else:
+                raise e
         click.echo(FeedbackManager.success(message="✓ Rows appended!"))
 
 
@@ -902,3 +913,33 @@ def generate_short_id():
 
 def generate_kafka_group_id(topic: str):
     return f"{topic}_{int(datetime.timestamp(datetime.now()))}"
+
+
+async def analyze_quarantine(datasource_name: str, project: Project, client: TinyB):
+    config = CLIConfig.get_project_config()
+    res = await client.query(
+        f"SELECT * FROM {datasource_name}_quarantine ORDER BY insertion_date DESC LIMIT 1 FORMAT JSON"
+    )
+    quarantine_data = res["data"]
+    error_message = json.dumps(res["data"])
+    user_token = config.get_user_token()
+    click.echo(FeedbackManager.gray(message=f"\n» Analyzing errors in {datasource_name}_quarantine..."))
+    if user_token:
+        llm = LLM(user_token=user_token, host=config.get_client().host)
+        ds_filenames = project.get_datasource_files()
+        datasource_definition = next(
+            (Path(f).read_text() for f in ds_filenames if f.endswith(f"{datasource_name}.datasource")), ""
+        )
+        response_llm = llm.ask(
+            system_prompt=quarantine_prompt(datasource_definition),
+            prompt=f"The quarantine errors are:\n{json.dumps(quarantine_data)}",
+        )
+        response = extract_xml(response_llm, "quarantine_errors")
+        error_message += "\n" + response
+        click.echo(response)
+    else:
+        echo_safe_humanfriendly_tables_format_smart_table(
+            data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
+        )
+
+    add_telemetry_event("datasource_error", error=f"quarantine_error: {error_message}")

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/tb/modules/feedback_manager.py

@@ -437,6 +437,9 @@ class FeedbackManager:
     error_tag_not_found = error_message("Tag {tag_name} not found.")
     error_build_failed = error_message("Build failed")
     error_request_failed = error_message("Request failed with status code {status_code}, please try again later.")
+    error_invalid_output_format = error_message(
+        "Invalid output format for this command. Supported formats are: {formats}"
+    )
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird/tb/modules/info.py

@@ -6,7 +6,7 @@ import click
 from tinybird.tb.client import TinyB
 from tinybird.tb.config import get_display_cloud_host
 from tinybird.tb.modules.cli import CLIConfig, cli
-from tinybird.tb.modules.common import coro, echo_json, format_robust_table
+from tinybird.tb.modules.common import coro, echo_json, force_echo, format_robust_table
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_tinybird_local_config
 from tinybird.tb.modules.project import Project
@@ -19,14 +19,19 @@ async def info(ctx: click.Context) -> None:
     """Get information about the project that is currently being used"""
     ctx_config = ctx.ensure_object(dict)["config"]
     project: Project = ctx.ensure_object(dict)["project"]
-    is_json = ctx.ensure_object(dict)["output"] == "json"
+    output = ctx.ensure_object(dict)["output"]
+
+    if output not in {"human", "json"}:
+        force_echo(FeedbackManager.error_invalid_output_format(formats=", ".join(["human", "json"])))
+        return
+
     click.echo(FeedbackManager.highlight(message="» Tinybird Cloud:"))
     cloud_table, cloud_columns = await get_cloud_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
     local_table, local_columns = await get_local_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Project:"))
     project_table, project_columns = await get_project_info(project.folder)
-    if is_json:
+    if output == "json":
         cloud_data = {}
         if cloud_columns and cloud_table and isinstance(cloud_table, list) and len(cloud_table) > 0:
             cloud_data = {column: cloud_table[0][i] for i, column in enumerate(cloud_columns)}

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev191}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev190
+Version: 0.0.1.dev191
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird