tinybird 0.0.1.dev190__tar.gz → 0.0.1.dev192__tar.gz

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev190
+Version: 0.0.1.dev192
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/prompts.py

@@ -1010,3 +1010,22 @@ Current README.md file:
 <readme>[readme content here]</readme>
 </readme_instructions>
 """
+
+
+def quarantine_prompt(datasource_definition: str) -> str:
+    return f"""
+- You are an expert in Tinybird.
+- You are given a list of rows that went to quarantine during ingestion because of data quality issues.
+- Return the errors in a human readable format so the user can understand what is the problem and fix it.
+- Be concise and to the point.
+- Do not mention clickhouse tables. Refer to data sources instead.
+- The possible fixes recommended fixes are:
+    - Changing a column type in the datasource definition: tell the user what to change in the datasource file, then build again before appending the data.
+    - Changing the data because of the wrong data type: tell the user what to change in the data file, then append the data again.
+- The format of the response always inside the tag <quarantine_errors>[response]</quarantine_errors>
+- Do not use markdown format in the response, because it is a CLI output.
+- The datasource definition is the following:
+<datasource_definition>
+{datasource_definition}
+</datasource_definition>
+"""

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev190'
-__revision__ = '8a00bcf'
+__version__ = '0.0.1.dev192'
+__revision__ = '116c2dc'

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/build.py

@@ -258,7 +258,6 @@ def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
     table_name = diff
     resource_path = Path(filename)
     resource_name = resource_path.stem
-    resource_content = resource_path.read_text()
 
     pipeline = resource_name if filename.endswith(".pipe") else None
 
@@ -269,7 +268,7 @@ def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
 
     res = asyncio.run(tb_client.query(sql, pipeline=pipeline))
     print_table_formatted(res, table_name)
-    if Project.is_endpoint(resource_content):
+    if Project.get_pipe_type(filename) == "endpoint":
         example_params = {
             "format": "json",
             "pipe": resource_name,

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/cli.py

@@ -28,7 +28,9 @@ from tinybird.tb.modules.common import (
     CLIException,
     _get_tb_client,
     coro,
+    echo_json,
     echo_safe_format_table,
+    force_echo,
     getenv_bool,
     try_update_config_with_remote,
 )
@@ -72,7 +74,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
 @click.option(
-    "--output", type=click.Choice(["json", "human"], case_sensitive=False), default="human", help="Output format"
+    "--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
 )
 @click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
 @click.version_option(version=VERSION)
@@ -95,7 +97,7 @@ async def cli(
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
     # We need to unpatch for our tests not to break
-    if output == "json":
+    if output != "human":
         __hide_click_output()
     else:
         if show_tokens or not cloud or ctx.invoked_subcommand == "build":
@@ -195,17 +197,10 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
 
 @cli.command()
 @click.argument("query", required=False)
-@click.option("--rows_limit", default=100, help="Max number of rows retrieved")
+@click.option("--rows-limit", default=100, help="Max number of rows retrieved")
 @click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
 @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
 @click.option("--node", default=None, help="The NODE name")
-@click.option(
-    "--format",
-    "format_",
-    type=click.Choice(["json", "csv", "human"], case_sensitive=False),
-    default="human",
-    help="Output format",
-)
 @click.option("--stats/--no-stats", default=False, help="Show query stats")
 @click.pass_context
 @coro
@@ -216,13 +211,13 @@ async def sql(
     pipeline: Optional[str],
     pipe: Optional[str],
     node: Optional[str],
-    format_: str,
     stats: bool,
 ) -> None:
     """Run SQL query over data sources and pipes."""
-
     client = ctx.ensure_object(dict)["client"]
-    req_format = "CSVWithNames" if format_ == "csv" else "JSON"
+    output = ctx.ensure_object(dict)["output"]
+
+    req_format = "CSVWithNames" if output == "csv" else "JSON"
     res = None
     try:
         if query:
@@ -275,11 +270,11 @@ async def sql(
         bytes_read = humanfriendly.format_size(stats_dict["bytes_read"])
         click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
 
-    if format_ == "csv":
-        click.echo(res)
+    if output == "csv":
+        force_echo(str(res))
     elif isinstance(res, dict) and "data" in res and res["data"]:
-        if format_ == "json":
-            click.echo(json.dumps(res, indent=8))
+        if output == "json":
+            echo_json(res, indent=8)
         else:
             dd = []
             for d in res["data"]:

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/common.py

@@ -2250,5 +2250,9 @@ def get_error_event(error: str) -> Tuple[str, str]:
     return error_event, silent_error_msg
 
 
-def echo_json(data: Dict[str, Any]) -> None:
-    click.echo(json.dumps(data), force_output=True)  # type: ignore
+def force_echo(string: str) -> None:
+    click.echo(string, force_output=True)  # type: ignore
+
+
+def echo_json(data: Dict[str, Any], indent: Union[None, int, str] = None) -> None:
+    force_echo(json.dumps(data, indent=indent))

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/datasource.py

@@ -18,6 +18,7 @@ import humanfriendly
 import requests
 from click import Context
 
+from tinybird.prompts import quarantine_prompt
 from tinybird.syncasync import sync_to_async
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.tb.modules.cli import cli
@@ -42,7 +43,10 @@ from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.llm import LLM
+from tinybird.tb.modules.llm_utils import extract_xml
 from tinybird.tb.modules.project import Project
+from tinybird.tb.modules.telemetry import add_telemetry_event
 
 
 @cli.group()
@@ -151,6 +155,7 @@ async def datasource_append(
     """
     env: str = ctx.ensure_object(dict)["env"]
     client: TinyB = ctx.obj["client"]
+    project: Project = ctx.ensure_object(dict)["project"]
 
     # If data is passed as argument, we detect if it's a JSON object, a URL or a file
     if data:
@@ -244,15 +249,10 @@ async def datasource_append(
 
     if events:
         click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
-        try:
-            json_data = json.loads(events)
-        except Exception:
-            raise CLIDatasourceException(FeedbackManager.error(message="Invalid events data"))
-
         response = await sync_to_async(requests.post)(
             f"{client.host}/v0/events?name={datasource_name}",
             headers={"Authorization": f"Bearer {client.token}"},
-            json=json_data,
+            data=events,
         )
 
         try:
@@ -269,21 +269,32 @@ async def datasource_append(
                 )
             )
         if quarantined_rows > 0:
-            raise CLIDatasourceException(
+            click.echo(
                 FeedbackManager.error(
-                    message=f"{quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
+                    message=f"✗ {quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
                 )
             )
+            await analyze_quarantine(datasource_name, project, client)
+            return
     else:
         click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
-        await push_data(
-            client,
-            datasource_name,
-            data,
-            mode="append",
-            concurrency=concurrency,
-            silent=True,
-        )
+        try:
+            await push_data(
+                client,
+                datasource_name,
+                data,
+                mode="append",
+                concurrency=concurrency,
+                silent=True,
+            )
+        except Exception as e:
+            is_quarantined = "quarantine" in str(e)
+            click.echo(FeedbackManager.error(message="✗ " + str(e)))
+            if is_quarantined:
+                await analyze_quarantine(datasource_name, project, client)
+                return
+            else:
+                raise e
         click.echo(FeedbackManager.success(message="✓ Rows appended!"))
 
 
@@ -902,3 +913,33 @@ def generate_short_id():
 
 def generate_kafka_group_id(topic: str):
     return f"{topic}_{int(datetime.timestamp(datetime.now()))}"
+
+
+async def analyze_quarantine(datasource_name: str, project: Project, client: TinyB):
+    config = CLIConfig.get_project_config()
+    res = await client.query(
+        f"SELECT * FROM {datasource_name}_quarantine ORDER BY insertion_date DESC LIMIT 1 FORMAT JSON"
+    )
+    quarantine_data = res["data"]
+    error_message = json.dumps(res["data"])
+    user_token = config.get_user_token()
+    click.echo(FeedbackManager.gray(message=f"\n» Analyzing errors in {datasource_name}_quarantine..."))
+    if user_token:
+        llm = LLM(user_token=user_token, host=config.get_client().host)
+        ds_filenames = project.get_datasource_files()
+        datasource_definition = next(
+            (Path(f).read_text() for f in ds_filenames if f.endswith(f"{datasource_name}.datasource")), ""
+        )
+        response_llm = llm.ask(
+            system_prompt=quarantine_prompt(datasource_definition),
+            prompt=f"The quarantine errors are:\n{json.dumps(quarantine_data)}",
+        )
+        response = extract_xml(response_llm, "quarantine_errors")
+        error_message += "\n" + response
+        click.echo(response)
+    else:
+        echo_safe_humanfriendly_tables_format_smart_table(
+            data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
+        )
+
+    add_telemetry_event("datasource_error", error=f"quarantine_error: {error_message}")

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/deployment.py

@@ -15,7 +15,7 @@ from tinybird.tb.modules.common import (
     get_display_cloud_host,
     sys_exit,
 )
-from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.feedback_manager import FeedbackManager, bcolors
 from tinybird.tb.modules.project import Project
 
 
@@ -229,10 +229,17 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
 
             last_deployment = result.get("deployment")
             if last_deployment.get("status") == "deleted":
-                click.echo(FeedbackManager.success(message="✓ Deployment is live!"))
+                click.echo(FeedbackManager.success(message=f"✓ Deployment #{candidate_deployment.get('id')} is live!"))
                 break
 
             time.sleep(5)
+    if last_deployment.get("id") == "0":
+        # This is the first deployment, so we prompt the user to ingest data
+        click.echo(
+            FeedbackManager.info(
+                message="A deployment with no data is useless. Learn how to ingest at https://www.tinybird.co/docs/forward/get-data-in"
+            )
+        )
 
 
 # TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for discarding a
@@ -590,8 +597,8 @@ def create_deployment(
         if status == "success":
             host = get_display_cloud_host(client.host)
             click.echo(
-                FeedbackManager.gray(message="Deployment URL: ")
-                + FeedbackManager.info(message=f"{host}/{config.get('name')}/deployments/{deployment.get('id')}")
+                FeedbackManager.info(message="Deployment URL: ")
+                + f"{bcolors.UNDERLINE}{host}/{config.get('name')}/deployments/{deployment.get('id')}{bcolors.ENDC}"
             )
 
             if wait:
@@ -655,40 +662,46 @@ def create_deployment(
 
 def print_changes(result: dict, project: Project) -> None:
     deployment = result.get("deployment", {})
-    resources_columns = ["status", "name", "path"]
+    resources_columns = ["status", "name", "type", "path"]
     resources: list[list[Union[str, None]]] = []
     tokens_columns = ["Change", "Token name", "Added permissions", "Removed permissions"]
     tokens: list[Tuple[str, str, str, str]] = []
 
     for ds in deployment.get("new_datasource_names", []):
-        resources.append(["new", ds, project.get_resource_path(ds, "datasource")])
+        resources.append(["new", ds, "datasource", project.get_resource_path(ds, "datasource")])
 
     for p in deployment.get("new_pipe_names", []):
-        resources.append(["new", p, project.get_resource_path(p, "pipe")])
+        path = str(project.get_resource_path(p, "pipe"))
+        pipe_type = project.get_pipe_type(path)
+        resources.append(["new", p, pipe_type, path])
 
     for dc in deployment.get("new_data_connector_names", []):
-        resources.append(["new", dc, project.get_resource_path(dc, "connection")])
+        resources.append(["new", dc, "connection", project.get_resource_path(dc, "connection")])
 
     for ds in deployment.get("changed_datasource_names", []):
-        resources.append(["modified", ds, project.get_resource_path(ds, "datasource")])
+        resources.append(["modified", ds, "datasource", project.get_resource_path(ds, "datasource")])
 
     for p in deployment.get("changed_pipe_names", []):
-        resources.append(["modified", p, project.get_resource_path(p, "pipe")])
+        path = str(project.get_resource_path(p, "pipe"))
+        pipe_type = project.get_pipe_type(path)
+        resources.append(["modified", p, pipe_type, path])
 
     for dc in deployment.get("changed_data_connector_names", []):
-        resources.append(["modified", dc, project.get_resource_path(dc, "connection")])
+        resources.append(["modified", dc, "connection", project.get_resource_path(dc, "connection")])
 
     for ds in deployment.get("disconnected_data_source_names", []):
-        resources.append(["modified", ds, project.get_resource_path(ds, "datasource")])
+        resources.append(["modified", ds, "datasource", project.get_resource_path(ds, "datasource")])
 
     for ds in deployment.get("deleted_datasource_names", []):
-        resources.append(["deleted", ds, project.get_resource_path(ds, "datasource")])
+        resources.append(["deleted", ds, "datasource", project.get_resource_path(ds, "datasource")])
 
     for p in deployment.get("deleted_pipe_names", []):
-        resources.append(["deleted", p, project.get_resource_path(p, "pipe")])
+        path = str(project.get_resource_path(p, "pipe"))
+        pipe_type = project.get_pipe_type(path)
+        resources.append(["deleted", p, pipe_type, path])
 
     for dc in deployment.get("deleted_data_connector_names", []):
-        resources.append(["deleted", dc, project.get_resource_path(dc, "connection")])
+        resources.append(["deleted", dc, "connection", project.get_resource_path(dc, "connection")])
 
     for token_change in deployment.get("token_changes", []):
         token_name = token_change.get("token_name")
@@ -707,9 +720,9 @@ def print_changes(result: dict, project: Project) -> None:
         click.echo(FeedbackManager.info(message="\n* Changes to be deployed:"))
         echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=resources_columns)
     else:
-        click.echo(FeedbackManager.info(message="\n* No changes to be deployed"))
+        click.echo(FeedbackManager.gray(message="\n* No changes to be deployed"))
     if tokens:
         click.echo(FeedbackManager.info(message="\n* Changes in tokens to be deployed:"))
         echo_safe_humanfriendly_tables_format_smart_table(tokens, column_names=tokens_columns)
     else:
-        click.echo(FeedbackManager.info(message="* No changes in tokens to be deployed"))
+        click.echo(FeedbackManager.gray(message="* No changes in tokens to be deployed"))

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/feedback_manager.py

@@ -437,6 +437,9 @@ class FeedbackManager:
     error_tag_not_found = error_message("Tag {tag_name} not found.")
     error_build_failed = error_message("Build failed")
     error_request_failed = error_message("Request failed with status code {status_code}, please try again later.")
+    error_invalid_output_format = error_message(
+        "Invalid output format for this command. Supported formats are: {formats}"
+    )
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/info.py

@@ -6,7 +6,7 @@ import click
 from tinybird.tb.client import TinyB
 from tinybird.tb.config import get_display_cloud_host
 from tinybird.tb.modules.cli import CLIConfig, cli
-from tinybird.tb.modules.common import coro, echo_json, format_robust_table
+from tinybird.tb.modules.common import coro, echo_json, force_echo, format_robust_table
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_tinybird_local_config
 from tinybird.tb.modules.project import Project
@@ -19,14 +19,19 @@ async def info(ctx: click.Context) -> None:
     """Get information about the project that is currently being used"""
     ctx_config = ctx.ensure_object(dict)["config"]
     project: Project = ctx.ensure_object(dict)["project"]
-    is_json = ctx.ensure_object(dict)["output"] == "json"
+    output = ctx.ensure_object(dict)["output"]
+
+    if output not in {"human", "json"}:
+        force_echo(FeedbackManager.error_invalid_output_format(formats=", ".join(["human", "json"])))
+        return
+
     click.echo(FeedbackManager.highlight(message="» Tinybird Cloud:"))
     cloud_table, cloud_columns = await get_cloud_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
     local_table, local_columns = await get_local_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Project:"))
     project_table, project_columns = await get_project_info(project.folder)
-    if is_json:
+    if output == "json":
         cloud_data = {}
         if cloud_columns and cloud_table and isinstance(cloud_table, list) and len(cloud_table) > 0:
             cloud_data = {column: cloud_table[0][i] for i, column in enumerate(cloud_columns)}

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird/tb/modules/project.py

@@ -51,7 +51,9 @@ class Project:
         return project_files
 
     def get_resource_path(self, resource_name: str, resource_type: str) -> Optional[str]:
-        full_path = next((p for p in self.get_project_files() if p.endswith(resource_name + f".{resource_type}")), "")
+        full_path = next(
+            (p for p in self.get_project_files() if p.endswith("/" + resource_name + f".{resource_type}")), ""
+        )
         if not full_path:
             return None
         return Path(full_path).relative_to(self.path).as_posix()
@@ -114,8 +116,15 @@ class Project:
         return datafiles
 
     @staticmethod
-    def is_endpoint(content: str) -> bool:
-        return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
+    def get_pipe_type(path: str) -> str:
+        content = Path(path).read_text()
+        if re.search(r"TYPE endpoint", content, re.IGNORECASE):
+            return "endpoint"
+        elif re.search(r"TYPE materialized", content, re.IGNORECASE):
+            return "materialization"
+        elif re.search(r"TYPE copy", content, re.IGNORECASE):
+            return "copy"
+        return "pipe"
 
     @staticmethod
     def is_kafka_connection(content: str) -> bool:

{tinybird-0.0.1.dev190 → tinybird-0.0.1.dev192}/tinybird.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev190
+Version: 0.0.1.dev192
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird