tinybird 0.0.1.dev189__py3-none-any.whl → 0.0.1.dev191__py3-none-any.whl

tinybird/prompts.py

@@ -1010,3 +1010,22 @@ Current README.md file:
 <readme>[readme content here]</readme>
 </readme_instructions>
 """
+
+
+def quarantine_prompt(datasource_definition: str) -> str:
+    return f"""
+- You are an expert in Tinybird.
+- You are given a list of rows that went to quarantine during ingestion because of data quality issues.
+- Return the errors in a human readable format so the user can understand what is the problem and fix it.
+- Be concise and to the point.
+- Do not mention clickhouse tables. Refer to data sources instead.
+- The possible fixes recommended fixes are:
+    - Changing a column type in the datasource definition: tell the user what to change in the datasource file, then build again before appending the data.
+    - Changing the data because of the wrong data type: tell the user what to change in the data file, then append the data again.
+- The format of the response always inside the tag <quarantine_errors>[response]</quarantine_errors>
+- Do not use markdown format in the response, because it is a CLI output.
+- The datasource definition is the following:
+<datasource_definition>
+{datasource_definition}
+</datasource_definition>
+"""

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev189'
-__revision__ = 'd55a4dc'
+__version__ = '0.0.1.dev191'
+__revision__ = 'c8535c7'

tinybird/tb/client.py

@@ -1024,11 +1024,7 @@ class TinyB:
         )
 
     async def kafka_list_topics(self, connection_id: str, timeout=5):
-        resp = await self._req(
-            f"/v0/connectors/{connection_id}/preview?preview_activity=false",
-            connect_timeout=timeout,
-            request_timeout=timeout,
-        )
+        resp = await self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
         return [x["topic"] for x in resp["preview"]]
 
     async def get_gcp_service_account_details(self) -> Dict[str, Any]:

tinybird/tb/modules/cli.py

@@ -28,7 +28,9 @@ from tinybird.tb.modules.common import (
     CLIException,
     _get_tb_client,
     coro,
+    echo_json,
     echo_safe_format_table,
+    force_echo,
     getenv_bool,
     try_update_config_with_remote,
 )
@@ -72,7 +74,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
 @click.option(
-    "--output", type=click.Choice(["json", "human"], case_sensitive=False), default="human", help="Output format"
+    "--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
 )
 @click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
 @click.version_option(version=VERSION)
@@ -95,7 +97,7 @@ async def cli(
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
     # We need to unpatch for our tests not to break
-    if output == "json":
+    if output != "human":
         __hide_click_output()
     else:
         if show_tokens or not cloud or ctx.invoked_subcommand == "build":
@@ -195,17 +197,10 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
 
 @cli.command()
 @click.argument("query", required=False)
-@click.option("--rows_limit", default=100, help="Max number of rows retrieved")
+@click.option("--rows-limit", default=100, help="Max number of rows retrieved")
 @click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
 @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
 @click.option("--node", default=None, help="The NODE name")
-@click.option(
-    "--format",
-    "format_",
-    type=click.Choice(["json", "csv", "human"], case_sensitive=False),
-    default="human",
-    help="Output format",
-)
 @click.option("--stats/--no-stats", default=False, help="Show query stats")
 @click.pass_context
 @coro
@@ -216,13 +211,13 @@ async def sql(
     pipeline: Optional[str],
     pipe: Optional[str],
     node: Optional[str],
-    format_: str,
     stats: bool,
 ) -> None:
     """Run SQL query over data sources and pipes."""
-
     client = ctx.ensure_object(dict)["client"]
-    req_format = "CSVWithNames" if format_ == "csv" else "JSON"
+    output = ctx.ensure_object(dict)["output"]
+
+    req_format = "CSVWithNames" if output == "csv" else "JSON"
     res = None
     try:
         if query:
@@ -275,11 +270,11 @@ async def sql(
         bytes_read = humanfriendly.format_size(stats_dict["bytes_read"])
         click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
 
-    if format_ == "csv":
-        click.echo(res)
+    if output == "csv":
+        force_echo(str(res))
     elif isinstance(res, dict) and "data" in res and res["data"]:
-        if format_ == "json":
-            click.echo(json.dumps(res, indent=8))
+        if output == "json":
+            echo_json(res, indent=8)
         else:
             dd = []
             for d in res["data"]:

tinybird/tb/modules/common.py

@@ -424,6 +424,14 @@ async def _analyze(filename: str, client: TinyB, format: str, connector: Optiona
     return meta, data
 
 
+async def analyze_file(filename: str, client: TinyB, format: str):
+    meta, data = await _analyze(filename, client, format)
+    schema = meta["analysis"]["schema"]
+    schema = schema.replace(", ", ",\n    ")
+    content = f"""DESCRIPTION >\n    Generated from {filename}\n\nSCHEMA >\n    {schema}"""
+    return content
+
+
 async def _generate_datafile(
     filename: str,
     client: TinyB,
@@ -971,7 +979,7 @@ async def push_data(
     cb.prev_done = 0  # type: ignore[attr-defined]
 
     if not silent:
-        click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name}..."))
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}..."))
 
     if isinstance(url, list):
         urls = url
@@ -1882,15 +1890,19 @@ def get_gcs_connection_name(project_folder) -> str:
     return get_connection_name(project_folder=project_folder, connection_type="GCS")
 
 
-def get_connection_name(project_folder: str, connection_type: str) -> str:
-    connection_name = None
+def get_kafka_connection_name(project_folder: str, connection_name: Optional[str] = None) -> str:
+    return get_connection_name(project_folder=project_folder, connection_type="KAFKA", connection_name=connection_name)
+
+
+def get_connection_name(project_folder: str, connection_type: str, connection_name: Optional[str] = None) -> str:
     valid_pattern = r"^[a-zA-Z][a-zA-Z0-9_]*$"
 
     while not connection_name:
+        short_id = str(uuid.uuid4())[:4]
         connection_name = click.prompt(
-            f"🔗 Enter a name for your new Tinybird {connection_type} connection (use alphanumeric characters, and underscores)",
-            prompt_suffix="\n> ",
+            "Enter a name (only alphanumeric characters and underscores)",
             show_default=True,
+            default=f"{connection_type.lower()}_{short_id}",
         )
         assert isinstance(connection_name, str)
 
@@ -2238,5 +2250,9 @@ def get_error_event(error: str) -> Tuple[str, str]:
     return error_event, silent_error_msg
 
 
-def echo_json(data: Dict[str, Any]) -> None:
-    click.echo(json.dumps(data), force_output=True)  # type: ignore
+def force_echo(string: str) -> None:
+    click.echo(string, force_output=True)  # type: ignore
+
+
+def echo_json(data: Dict[str, Any], indent: Union[None, int, str] = None) -> None:
+    force_echo(json.dumps(data, indent=indent))

tinybird/tb/modules/connection.py

@@ -18,6 +18,7 @@ from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     get_gcs_connection_name,
     get_gcs_svc_account_creds,
+    get_kafka_connection_name,
     get_s3_connection_name,
     production_aws_iamrole_only,
     run_aws_iamrole_connection_flow,
@@ -26,6 +27,7 @@ from tinybird.tb.modules.common import (
 from tinybird.tb.modules.create import (
     generate_aws_iamrole_connection_file_with_secret,
     generate_gcs_connection_file_with_secrets,
+    generate_kafka_connection_with_secrets,
 )
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
@@ -275,3 +277,21 @@ async def connection_create_gcs(ctx: Context) -> None:
             connection_path=connection_path,
         )
     )
+
+
+@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
+@click.option("--name", help="The name of the connection")
+@click.pass_context
+@coro
+async def connection_create_kafka(ctx: Context, name: Optional[str] = None) -> None:
+    """
+    Creates a Kafka connection.
+
+    \b
+    $ tb connection create kafka
+    """
+    click.echo(FeedbackManager.highlight(message="» Creating Kafka connection..."))
+    project: Project = ctx.ensure_object(dict)["project"]
+    name = get_kafka_connection_name(project.folder, name)
+    await generate_kafka_connection_with_secrets(name=name, folder=project.folder)
+    click.echo(FeedbackManager.success(message="✓ Done!"))

tinybird/tb/modules/create.py

@@ -409,10 +409,14 @@ def generate_connection_file(name: str, content: str, folder: str, skip_feedback
 
 
 async def generate_aws_iamrole_connection_file_with_secret(
-    name: str, service: str, role_arn_secret_name: str, region: str, folder: str
+    name: str, service: str, role_arn_secret_name: str, region: str, folder: str, with_default_secret: bool = False
 ) -> Path:
+    if with_default_secret:
+        default_secret = ', "arn:aws:iam::123456789012:role/my-role"'
+    else:
+        default_secret = ""
     content = f"""TYPE {service}
-S3_ARN {{{{ tb_secret("{role_arn_secret_name}") }}}}
+S3_ARN {{{{ tb_secret("{role_arn_secret_name}"{default_secret}) }}}}
 S3_REGION {region}
 """
     file_path = generate_connection_file(name, content, folder, skip_feedback=True)
@@ -483,6 +487,7 @@ async def create_resources_from_data(
     data: str,
     project: Project,
     config: Dict[str, Any],
+    skip_pipes: bool = False,
 ) -> List[Path]:
     local_client = await get_tinybird_local_client(config)
     folder_path = project.path
@@ -495,7 +500,7 @@ async def create_resources_from_data(
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -510,7 +515,9 @@ TYPE ENDPOINT
     return result
 
 
-async def create_resources_from_url(url: str, project: Project, config: Dict[str, Any]) -> List[Path]:
+async def create_resources_from_url(
+    url: str, project: Project, config: Dict[str, Any], skip_pipes: bool = False
+) -> List[Path]:
     result: List[Path] = []
     local_client = await get_tinybird_local_client(config)
     format = url.split(".")[-1]
@@ -518,7 +525,7 @@ async def create_resources_from_url(url: str, project: Project, config: Dict[str
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -531,3 +538,14 @@ TYPE ENDPOINT
         )
         result.append(pipe_file)
     return result
+
+
+async def generate_kafka_connection_with_secrets(name: str, folder: str) -> Path:
+    content = """TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("KAFKA_SERVERS", "localhost:9092") }}
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY {{ tb_secret("KAFKA_USERNAME", "") }}
+KAFKA_SECRET {{ tb_secret("KAFKA_PASSWORD", "") }}
+"""
+    return generate_connection_file(name, content, folder, skip_feedback=True)

tinybird/tb/modules/datasource.py

@@ -7,27 +7,46 @@ import asyncio
 import json
 import os
 import re
+import uuid
+from datetime import datetime
+from pathlib import Path
 from typing import Optional
+from urllib.parse import urlparse
 
 import click
 import humanfriendly
+import requests
 from click import Context
 
+from tinybird.prompts import quarantine_prompt
+from tinybird.syncasync import sync_to_async
 from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import (
     _analyze,
+    analyze_file,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
     get_format_from_filename_or_url,
     load_connector_config,
+    normalize_datasource_name,
     push_data,
 )
+from tinybird.tb.modules.config import CLIConfig
+from tinybird.tb.modules.create import (
+    create_resources_from_prompt,
+    generate_aws_iamrole_connection_file_with_secret,
+    generate_gcs_connection_file_with_secrets,
+    generate_kafka_connection_with_secrets,
+)
 from tinybird.tb.modules.datafile.common import get_name_version
 from tinybird.tb.modules.datafile.fixture import persist_fixture
 from tinybird.tb.modules.exceptions import CLIDatasourceException
 from tinybird.tb.modules.feedback_manager import FeedbackManager
+from tinybird.tb.modules.llm import LLM
+from tinybird.tb.modules.llm_utils import extract_xml
 from tinybird.tb.modules.project import Project
+from tinybird.tb.modules.telemetry import add_telemetry_event
 
 
 @cli.group()
@@ -107,33 +126,176 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
 
 
 @datasource.command(name="append")
-@click.argument("datasource_name", required=True)
-@click.argument("url", nargs=-1, required=True)
+@click.argument("datasource_name", required=False)
+@click.argument("data", nargs=-1, required=False)
+@click.option("--url", type=str, help="URL to append data from")
+@click.option("--file", type=str, help="Local file to append data from")
+@click.option("--events", type=str, help="Events to append data from")
 @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
 @click.pass_context
 @coro
 async def datasource_append(
     ctx: Context,
     datasource_name: str,
-    url,
+    data: Optional[str],
+    url: str,
+    file: str,
+    events: str,
     concurrency: int,
 ):
     """
     Appends data to an existing data source from URL, local file  or a connector
 
-    - Load from URL `tb datasource append [datasource_name] https://url_to_csv`
+    - Events API: `tb datasource append [datasource_name] --events '{"a":"b, "c":"d"}'`\n
+    - Local File: `tb datasource append [datasource_name] --file /path/to/local/file`\n
+    - Remote URL: `tb datasource append [datasource_name] --url https://url_to_csv`\n
+    - Kafka, S3 and GCS: https://www.tinybird.co/docs/forward/get-data-in/connectors\n
 
-    - Load from local file `tb datasource append [datasource_name] /path/to/local/file`
+    More info: https://www.tinybird.co/docs/forward/get-data-in
     """
-
+    env: str = ctx.ensure_object(dict)["env"]
     client: TinyB = ctx.obj["client"]
-    await push_data(
-        client,
-        datasource_name,
-        url,
-        mode="append",
-        concurrency=concurrency,
-    )
+    project: Project = ctx.ensure_object(dict)["project"]
+
+    # If data is passed as argument, we detect if it's a JSON object, a URL or a file
+    if data:
+        try:
+            json.loads(data)
+            events = data
+        except Exception:
+            pass
+        if urlparse(data).scheme in ("http", "https"):
+            url = data
+        if not events and not url:
+            file = data
+
+    # If data is not passed as argument, we use the data from the options
+    if not data:
+        data = file or url or events
+
+    if env == "local":
+        tip = "Did you build your project? Run `tb build` first."
+    else:
+        tip = "Did you deploy your project? Run `tb --cloud deploy` first."
+
+    datasources = await client.datasources()
+    if not datasources:
+        raise CLIDatasourceException(FeedbackManager.error(message=f"No data sources found. {tip}"))
+
+    if datasource_name and datasource_name not in [ds["name"] for ds in datasources]:
+        raise CLIDatasourceException(FeedbackManager.error(message=f"Datasource {datasource_name} not found. {tip}"))
+
+    if not datasource_name:
+        datasource_index = -1
+
+        click.echo(FeedbackManager.info(message="\n? Which data source do you want to ingest data into?"))
+        while datasource_index == -1:
+            for index, datasource in enumerate(datasources):
+                click.echo(f"  [{index + 1}] {datasource['name']}")
+            click.echo(
+                FeedbackManager.gray(message="Tip: Run tb datasource append [datasource_name] to skip this step.")
+            )
+
+            datasource_index = click.prompt("\nSelect option", default=1)
+
+            if datasource_index == 0:
+                click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
+                return None
+
+            try:
+                datasource_name = datasources[int(datasource_index) - 1]["name"]
+            except Exception:
+                datasource_index = -1
+
+    if not datasource_name:
+        raise CLIDatasourceException(FeedbackManager.error_datasource_name())
+
+    if not data:
+        data_index = -1
+        options = (
+            "Events API",
+            "Local File",
+            "Remote URL",
+        )
+        click.echo(FeedbackManager.info(message="\n? How do you want to ingest data?"))
+        while data_index == -1:
+            for index, option in enumerate(options):
+                click.echo(f"  [{index + 1}] {option}")
+            click.echo(
+                FeedbackManager.gray(
+                    message="Tip: Run tb datasource append [datasource_name] --events | --file | --url to skip this step"
+                )
+            )
+
+            data_index = click.prompt("\nSelect option", default=1)
+
+            if data_index == 0:
+                click.echo(FeedbackManager.warning(message="Data selection cancelled by user"))
+                return None
+
+            try:
+                data_index = int(data_index)
+            except Exception:
+                data_index = -1
+
+        if data_index == 1:
+            events = click.prompt("Events data")
+        elif data_index == 2:
+            data = click.prompt("Path to local file")
+        elif data_index == 3:
+            data = click.prompt("URL to remote file")
+        else:
+            raise CLIDatasourceException(FeedbackManager.error(message="Invalid ingestion option"))
+
+    if events:
+        click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
+        response = await sync_to_async(requests.post)(
+            f"{client.host}/v0/events?name={datasource_name}",
+            headers={"Authorization": f"Bearer {client.token}"},
+            data=events,
+        )
+
+        try:
+            res = response.json()
+        except Exception:
+            raise CLIDatasourceException(FeedbackManager.error(message=response.text))
+
+        successful_rows = res["successful_rows"]
+        quarantined_rows = res["quarantined_rows"]
+        if successful_rows > 0:
+            click.echo(
+                FeedbackManager.success(
+                    message=f"✓ {successful_rows} row{'' if successful_rows == 1 else 's'} appended!"
+                )
+            )
+        if quarantined_rows > 0:
+            click.echo(
+                FeedbackManager.error(
+                    message=f"✗ {quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
+                )
+            )
+            await analyze_quarantine(datasource_name, project, client)
+            return
+    else:
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
+        try:
+            await push_data(
+                client,
+                datasource_name,
+                data,
+                mode="append",
+                concurrency=concurrency,
+                silent=True,
+            )
+        except Exception as e:
+            is_quarantined = "quarantine" in str(e)
+            click.echo(FeedbackManager.error(message="✗ " + str(e)))
+            if is_quarantined:
+                await analyze_quarantine(datasource_name, project, client)
+                return
+            else:
+                raise e
+        click.echo(FeedbackManager.success(message="✓ Rows appended!"))
 
 
 @datasource.command(name="replace")
@@ -473,3 +635,311 @@ async def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
         raise
     except Exception as e:
         raise CLIDatasourceException(FeedbackManager.error_syncing_datasource(datasource=datasource_name, error=str(e)))
+
+
+@datasource.command(name="create")
+@click.option("--name", type=str, help="Name of the data source")
+@click.option("--blank", is_flag=True, default=False, help="Create a blank data source")
+@click.option("--file", type=str, help="Create a data source from a local file")
+@click.option("--url", type=str, help="Create a data source from a remote URL")
+@click.option("--connection", type=str, help="Create a data source from a connection")
+@click.option("--prompt", type=str, help="Create a data source from a prompt")
+@click.option("--s3", is_flag=True, default=False, help="Create a data source from a S3 connection")
+@click.option("--gcs", is_flag=True, default=False, help="Create a data source from a GCS connection")
+@click.option("--kafka", is_flag=True, default=False, help="Create a data source from a Kafka connection")
+@click.pass_context
+@coro
+async def datasource_create(
+    ctx: Context,
+    name: str,
+    blank: bool,
+    file: str,
+    url: str,
+    connection: str,
+    prompt: str,
+    s3: bool,
+    gcs: bool,
+    kafka: bool,
+):
+    project: Project = ctx.ensure_object(dict)["project"]
+    client: TinyB = ctx.ensure_object(dict)["client"]
+    config = ctx.ensure_object(dict)["config"]
+    env: str = ctx.ensure_object(dict)["env"]
+
+    if env == "cloud":
+        raise CLIDatasourceException(
+            FeedbackManager.error(message="`tb datasource create` is not available against Tinybird Cloud.")
+        )
+
+    datasource_types = (
+        "Blank",
+        "Local file",
+        "Remote URL",
+        "Kafka",
+        "S3",
+        "GCS",
+    )
+    datasource_type: Optional[str] = None
+    connection_file: Optional[str] = None
+    ds_content = ""
+
+    if file:
+        datasource_type = "Local file"
+    elif url:
+        datasource_type = "Remote URL"
+    elif blank:
+        datasource_type = "Blank"
+    elif connection:
+        connection_files = project.get_connection_files()
+        connection_file = next((f for f in connection_files if f.endswith(f"{connection}.connection")), None)
+        if connection_file:
+            connection_content = Path(connection_file).read_text()
+            if project.is_kafka_connection(connection_content):
+                datasource_type = "Kafka"
+            elif project.is_s3_connection(connection_content):
+                datasource_type = "S3"
+            elif project.is_gcs_connection(connection_content):
+                datasource_type = "GCS"
+    elif s3:
+        datasource_type = "S3"
+    elif gcs:
+        datasource_type = "GCS"
+    elif kafka:
+        datasource_type = "Kafka"
+    elif prompt:
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        user_token = config.get("user_token")
+        if not user_token:
+            raise Exception("This action requires authentication. Run 'tb login' first.")
+        project_config = CLIConfig.get_project_config()
+        tb_client: TinyB = project_config.get_client(token=config.get("token"), host=config.get("host"))
+        await create_resources_from_prompt(tb_client, user_token, prompt, project)
+        click.echo(FeedbackManager.success(message="✓ .datasource created!"))
+        return
+
+    if datasource_type is None:
+        click.echo(FeedbackManager.highlight(message="? Where do you want to create your .datasource from?"))
+        datasource_type_index = -1
+
+        while datasource_type_index == -1:
+            for index, datasource_type in enumerate(datasource_types):
+                click.echo(f"  [{index + 1}] {datasource_type}")
+            click.echo(
+                FeedbackManager.gray(
+                    message="Tip: Run `tb datasource create --file | --url | --connection` to skip this step."
+                )
+            )
+            datasource_type_index = click.prompt("\nSelect option", default=1)
+
+            if datasource_type_index == 0:
+                click.echo(FeedbackManager.warning(message="Datasource type selection cancelled by user"))
+                return None
+
+            try:
+                datasource_type = datasource_types[int(datasource_type_index) - 1]
+            except Exception:
+                datasource_type_index = -1
+
+    if not datasource_type:
+        click.echo(
+            FeedbackManager.error(
+                message=f"Invalid option: {datasource_type_index}. Please select a valid option from the list above."
+            )
+        )
+        return
+
+    connection_required = datasource_type in ("Kafka", "S3", "GCS")
+
+    if connection_required:
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        connection_type = datasource_type.lower()
+
+        def get_connection_files():
+            connection_files = []
+            if connection_type == "kafka":
+                connection_files = project.get_kafka_connection_files()
+            elif connection_type == "s3":
+                connection_files = project.get_s3_connection_files()
+            elif connection_type == "gcs":
+                connection_files = project.get_gcs_connection_files()
+            return connection_files
+
+        connection_files = get_connection_files()
+        if len(connection_files) == 0:
+            click.echo(FeedbackManager.error(message=f"x No {datasource_type} connections found."))
+            if click.confirm(
+                FeedbackManager.highlight(message=f"\n? Do you want to create a {datasource_type} connection? [Y/n]"),
+                show_default=False,
+                default=True,
+            ):
+                click.echo(FeedbackManager.gray(message="\n» Creating .connection file..."))
+                default_connection_name = f"{datasource_type.lower()}_{generate_short_id()}"
+                connection_name = click.prompt(
+                    FeedbackManager.highlight(message=f"? Connection name [{default_connection_name}]"),
+                    show_default=False,
+                    default=default_connection_name,
+                )
+                if datasource_type == "Kafka":
+                    await generate_kafka_connection_with_secrets(connection_name, folder=project.folder)
+                elif datasource_type == "S3":
+                    await generate_aws_iamrole_connection_file_with_secret(
+                        connection_name,
+                        service="s3",
+                        role_arn_secret_name="S3_ARN",
+                        region="eu-west-1",
+                        folder=project.folder,
+                        with_default_secret=True,
+                    )
+                elif datasource_type == "GCS":
+                    await generate_gcs_connection_file_with_secrets(
+                        connection_name,
+                        service="gcs",
+                        svc_account_creds="GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
+                        folder=project.folder,
+                    )
+                click.echo(FeedbackManager.info(message=f"/connections/{connection_name}.connection"))
+                click.echo(FeedbackManager.success(message="✓ .connection created!"))
+                connection_files = get_connection_files()
+            else:
+                click.echo(
+                    FeedbackManager.info(message=f"→ Run `tb connection create {datasource_type.lower()}` to add one.")
+                )
+                return
+
+        if not connection_file:
+            connection_file = connection_files[0]
+            connection_path = Path(connection_file)
+            connection = connection_path.stem
+
+    ds_content = """SCHEMA >
+`timestamp` DateTime `json:$.timestamp`,
+`session_id` String `json:$.session_id`
+"""
+
+    if datasource_type == "Local file":
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        if not file:
+            file = click.prompt(FeedbackManager.highlight(message="? Path"))
+            if file.startswith("~"):
+                file = os.path.expanduser(file)
+
+        folder_path = project.path
+        path = folder_path / file
+        if not path.exists():
+            path = Path(file)
+
+        data_format = path.suffix.lstrip(".")
+        ds_content = await analyze_file(str(path), client, format=data_format)
+        default_name = normalize_datasource_name(path.stem)
+        name = name or click.prompt(
+            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+            default=default_name,
+            show_default=False,
+        )
+
+    if datasource_type == "Remote URL":
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+        if not url:
+            url = click.prompt(FeedbackManager.highlight(message="? URL"))
+        format = url.split(".")[-1]
+        ds_content = await analyze_file(url, client, format)
+        default_name = normalize_datasource_name(Path(url).stem)
+        name = name or click.prompt(
+            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+            default=default_name,
+            show_default=False,
+        )
+
+    if datasource_type == "Blank":
+        click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
+
+    if datasource_type not in ("Remote URL", "Local file"):
+        default_name = f"ds_{generate_short_id()}"
+        name = name or click.prompt(
+            FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
+            default=default_name,
+            show_default=False,
+        )
+
+    if datasource_type == "Kafka":
+        connections = await client.connections("kafka")
+        connection_id = next((c["id"] for c in connections if c["name"] == connection), connection)
+        try:
+            topics = await client.kafka_list_topics(connection_id) if connection_id else []
+        except Exception:
+            topics = []
+        topic = topics[0] if len(topics) > 0 else "topic_0"
+        group_id = generate_kafka_group_id(topic)
+        ds_content += f"""
+KAFKA_CONNECTION_NAME {connection}
+KAFKA_TOPIC {{{{ tb_secret("KAFKA_TOPIC", "{topic}") }}}}
+KAFKA_GROUP_ID {{{{ tb_secret("KAFKA_GROUP_ID", "{group_id}") }}}}
+"""
+
+    if datasource_type == "S3":
+        if not connection:
+            connections = await client.connections("s3")
+            connection = next((c["name"] for c in connections if c["name"] == connection), connection)
+        ds_content += f"""
+IMPORT_CONNECTION_NAME "{connection}"
+IMPORT_BUCKET_URI "s3://my-bucket/*.csv"
+IMPORT_SCHEDULE "@auto"
+"""
+
+    if datasource_type == "GCS":
+        if not connection:
+            connections = await client.connections("gcs")
+            connection = next((c["name"] for c in connections if c["name"] == connection), connection)
+        ds_content += f"""
+IMPORT_CONNECTION_NAME "{connection}"
+IMPORT_BUCKET_URI "gs://my-bucket/*.csv"
+IMPORT_SCHEDULE "@auto"
+"""
+
+    click.echo(FeedbackManager.info(message=f"/datasources/{name}.datasource"))
+    datasources_path = project.path / "datasources"
+    if not datasources_path.exists():
+        datasources_path.mkdir()
+    ds_file = datasources_path / f"{name}.datasource"
+    if not ds_file.exists():
+        ds_file.touch()
+    ds_file.write_text(ds_content)
+    click.echo(FeedbackManager.success(message="✓ .datasource created!"))
+
+
+def generate_short_id():
+    return str(uuid.uuid4())[:4]
+
+
+def generate_kafka_group_id(topic: str):
+    return f"{topic}_{int(datetime.timestamp(datetime.now()))}"
+
+
+async def analyze_quarantine(datasource_name: str, project: Project, client: TinyB):
+    config = CLIConfig.get_project_config()
+    res = await client.query(
+        f"SELECT * FROM {datasource_name}_quarantine ORDER BY insertion_date DESC LIMIT 1 FORMAT JSON"
+    )
+    quarantine_data = res["data"]
+    error_message = json.dumps(res["data"])
+    user_token = config.get_user_token()
+    click.echo(FeedbackManager.gray(message=f"\n» Analyzing errors in {datasource_name}_quarantine..."))
+    if user_token:
+        llm = LLM(user_token=user_token, host=config.get_client().host)
+        ds_filenames = project.get_datasource_files()
+        datasource_definition = next(
+            (Path(f).read_text() for f in ds_filenames if f.endswith(f"{datasource_name}.datasource")), ""
+        )
+        response_llm = llm.ask(
+            system_prompt=quarantine_prompt(datasource_definition),
+            prompt=f"The quarantine errors are:\n{json.dumps(quarantine_data)}",
+        )
+        response = extract_xml(response_llm, "quarantine_errors")
+        error_message += "\n" + response
+        click.echo(response)
+    else:
+        echo_safe_humanfriendly_tables_format_smart_table(
+            data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
+        )
+
+    add_telemetry_event("datasource_error", error=f"quarantine_error: {error_message}")

tinybird/tb/modules/feedback_manager.py

@@ -437,6 +437,9 @@ class FeedbackManager:
     error_tag_not_found = error_message("Tag {tag_name} not found.")
     error_build_failed = error_message("Build failed")
     error_request_failed = error_message("Request failed with status code {status_code}, please try again later.")
+    error_invalid_output_format = error_message(
+        "Invalid output format for this command. Supported formats are: {formats}"
+    )
 
     info_incl_relative_path = info_message("** Relative path {path} does not exist, skipping.")
     info_ignoring_incl_file = info_message(

tinybird/tb/modules/info.py

@@ -6,7 +6,7 @@ import click
 from tinybird.tb.client import TinyB
 from tinybird.tb.config import get_display_cloud_host
 from tinybird.tb.modules.cli import CLIConfig, cli
-from tinybird.tb.modules.common import coro, echo_json, format_robust_table
+from tinybird.tb.modules.common import coro, echo_json, force_echo, format_robust_table
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_tinybird_local_config
 from tinybird.tb.modules.project import Project
@@ -19,14 +19,19 @@ async def info(ctx: click.Context) -> None:
     """Get information about the project that is currently being used"""
     ctx_config = ctx.ensure_object(dict)["config"]
     project: Project = ctx.ensure_object(dict)["project"]
-    is_json = ctx.ensure_object(dict)["output"] == "json"
+    output = ctx.ensure_object(dict)["output"]
+
+    if output not in {"human", "json"}:
+        force_echo(FeedbackManager.error_invalid_output_format(formats=", ".join(["human", "json"])))
+        return
+
     click.echo(FeedbackManager.highlight(message="» Tinybird Cloud:"))
     cloud_table, cloud_columns = await get_cloud_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
     local_table, local_columns = await get_local_info(ctx_config)
     click.echo(FeedbackManager.highlight(message="\n» Project:"))
     project_table, project_columns = await get_project_info(project.folder)
-    if is_json:
+    if output == "json":
         cloud_data = {}
         if cloud_columns and cloud_table and isinstance(cloud_table, list) and len(cloud_table) > 0:
             cloud_data = {column: cloud_table[0][i] for i, column in enumerate(cloud_columns)}

tinybird/tb/modules/project.py

@@ -77,6 +77,15 @@ class Project:
     def get_connection_files(self) -> List[str]:
         return self.get_files("connection")
 
+    def get_kafka_connection_files(self) -> List[str]:
+        return [f for f in self.get_connection_files() if self.is_kafka_connection(Path(f).read_text())]
+
+    def get_s3_connection_files(self) -> List[str]:
+        return [f for f in self.get_connection_files() if self.is_s3_connection(Path(f).read_text())]
+
+    def get_gcs_connection_files(self) -> List[str]:
+        return [f for f in self.get_connection_files() if self.is_gcs_connection(Path(f).read_text())]
+
     def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
         try:
             return parse_pipe(filename).datafile
@@ -107,3 +116,27 @@ class Project:
     @staticmethod
     def is_endpoint(content: str) -> bool:
         return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_kafka_connection(content: str) -> bool:
+        return re.search(r"TYPE kafka", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_s3_connection(content: str) -> bool:
+        return re.search(r"TYPE s3", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_gcs_connection(content: str) -> bool:
+        return re.search(r"TYPE gcs", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_kafka_datasource(content: str) -> bool:
+        return re.search(r"KAFKA_CONNECTION_NAME", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_s3_datasource(content: str) -> bool:
+        return re.search(r"IMPORT_CONNECTION_NAME", content, re.IGNORECASE) is not None
+
+    @staticmethod
+    def is_gcs_datasource(content: str) -> bool:
+        return re.search(r"IMPORT_CONNECTION_NAME", content, re.IGNORECASE) is not None

{tinybird-0.0.1.dev189.dist-info → tinybird-0.0.1.dev191.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev189
+Version: 0.0.1.dev191
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev189.dist-info → tinybird-0.0.1.dev191.dist-info}/RECORD

@@ -3,7 +3,7 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
 tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
 tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
 tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
-tinybird/prompts.py,sha256=uFzlVsGgWFwEOBhXkaLL4zY-wz8jCQgTFiMEfoFcgro,36550
+tinybird/prompts.py,sha256=rpgvMpR103niDnoDMSz8hRAYBdgfrorfD5-7g23BMQQ,37596
 tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
 tinybird/sql_template.py,sha256=hWW8JawSWLl9GeWPYkC_Yrxj7P0MHEVMJ0Px9bedEgM,99817
 tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
@@ -12,27 +12,27 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
 tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
 tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
 tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
-tinybird/tb/__cli__.py,sha256=Jw96cYl5Uk5poOJYvRJJfYGQbYLgCXUaCwjiBxNu3Zg,247
+tinybird/tb/__cli__.py,sha256=IuO2njWcdIXnuscWmXWEMTbCM9GnrTyYky59OtsEi3I,247
 tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
 tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
-tinybird/tb/client.py,sha256=59GH0IoYSV_KUG0eEbDDYHSWH4OlkVnSRFXE3mYAM0s,56571
+tinybird/tb/client.py,sha256=CO-dQw8h28X6T6IO-Z79yPBKaJQT1Rwya5b6gexvw58,56491
 tinybird/tb/config.py,sha256=jT9xndpeCY_g0HdB5qE2EquC0TFRRnkPnQFWZWd04jo,3998
 tinybird/tb/modules/build.py,sha256=KvF0s8hGgY_rZs7jSqYiauCk3MAlCmW_gQtnsJDJWBk,19411
 tinybird/tb/modules/cicd.py,sha256=Njb6eZOHHbUkoJJx6KoixO9PsfA_T-3Ybkya9-50Ca8,7328
-tinybird/tb/modules/cli.py,sha256=dXZs-MuqYPvxStVj7aLg36LwXtEB8NzTobDmHV9nzZI,15508
-tinybird/tb/modules/common.py,sha256=DYCjpj0iBaCDZ8BJ0MNG_6m6NyFMCrpQShIajHKLIfM,83373
+tinybird/tb/modules/cli.py,sha256=LW77oBqvZ6QB5X96158Kp13Tl24vKdxlyWvsSawpmCo,15399
+tinybird/tb/modules/common.py,sha256=2NRDRll0czmYjwLh3qv3DYL9aP8XgRkRAv5S3meCGfM,84062
 tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
-tinybird/tb/modules/connection.py,sha256=7oOR7x4PhBcm1ETFFCH2YJ_3oeGXjAbmx1cnZX9_L70,9014
+tinybird/tb/modules/connection.py,sha256=z1xWP2gtjKEbjc4ZF1aD7QUgl8V--wf2IRNy-4sRFm8,9779
 tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
-tinybird/tb/modules/create.py,sha256=sfIOcN3tujt7O1r9RNWqhhI-gQTDnO6zEgMwZHH2D8s,20201
-tinybird/tb/modules/datasource.py,sha256=0_6Cn07p5GoNBBGdu88pSeLvTWojln1-k23FsS8jTDs,17801
+tinybird/tb/modules/create.py,sha256=2uW-4t7c7e4xkZ-GpK_8XaA-nuXwklq7rTks4k6qrtI,20917
+tinybird/tb/modules/datasource.py,sha256=Uomj3bpVL5m4A0_ezWFXvQAjVER4EfmDz5gFkfGR7eo,37027
 tinybird/tb/modules/deployment.py,sha256=OGJdriqywhCtsG7rKLFtVSLjoEbbva1Nb29-jQBk3wM,27432
 tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
 tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
 tinybird/tb/modules/endpoint.py,sha256=XySDt3pk66vxOZ0egUfz4bY8bEk3BjOXkv-L0OIJ3sc,12083
 tinybird/tb/modules/exceptions.py,sha256=5jK91w1LPmtqIUfDpHe_Op5OxGz8-p1BPgtLREMIni0,5217
-tinybird/tb/modules/feedback_manager.py,sha256=sWJYBIZDPYLFWMKPv7FoVLbQxE7xx85bn5SbK9yYXoE,76782
-tinybird/tb/modules/info.py,sha256=iKeFbFkos7vYaBU7Vr5SI-fa1x7AbuUHB748jsGsaA4,5944
+tinybird/tb/modules/feedback_manager.py,sha256=cnMGTCXa04Zmwq0tPHuAeq_hxqTz3F8c7mXqTJ-MZsw,76920
+tinybird/tb/modules/info.py,sha256=qsXr2g45jRv2xD4Kt2sgsT0AR8sjwrAZf2s7mI-zxL4,6113
 tinybird/tb/modules/infra.py,sha256=fve30Gj3mG9zbquGxS2e4ipcOYOxviWQCpNFfEzJN_Q,33195
 tinybird/tb/modules/job.py,sha256=AsUCRNzy7HG5oJ4fyk9NpIm5NtNJgBZSy8MtJdXBe5A,3167
 tinybird/tb/modules/llm.py,sha256=KfsCYmKeW1VQz0iDZhGKCRkQv_Y3kTHh6JuxvofOguE,1076
@@ -45,7 +45,7 @@ tinybird/tb/modules/materialization.py,sha256=QJX5kCPhhm6IXBO1JsalVfbQdypCe_eOUD
 tinybird/tb/modules/mock.py,sha256=IyHweMUM6bUH8IhyiX2tTMpdVpTFUeAJ41lZ5P42-HQ,5303
 tinybird/tb/modules/open.py,sha256=OuctINN77oexpSjth9uoIZPCelKO4Li-yyVxeSnk1io,1371
 tinybird/tb/modules/pipe.py,sha256=AQKEDagO6e3psPVjJkS_MDbn8aK-apAiLp26k7jgAV0,2432
-tinybird/tb/modules/project.py,sha256=wOL0vmkbM6Jhxbv0zYwt3JjAoRQDvkbbus01bpzfhA0,3846
+tinybird/tb/modules/project.py,sha256=iIEaBQsdLXyzJ_45Paf1jwbSrtTwWv131VCuPsTQttA,5215
 tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb/modules/secret.py,sha256=WsqzxxLh9W_jkuHL2JofMXdIJy0lT5WEI-7bQSIDgAc,2921
 tinybird/tb/modules/shell.py,sha256=Zd_4Ak_5tKVX-cw6B4ag36xZeEGHeh-jZpAsIXkoMoE,14116
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
 tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
 tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
 tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
-tinybird-0.0.1.dev189.dist-info/METADATA,sha256=fz7av4cjpbqq8H_bP6CsG5NpL3Dg9iBRFHEG1U7luSs,1608
-tinybird-0.0.1.dev189.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-tinybird-0.0.1.dev189.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
-tinybird-0.0.1.dev189.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
-tinybird-0.0.1.dev189.dist-info/RECORD,,
+tinybird-0.0.1.dev191.dist-info/METADATA,sha256=6DYh0XzR-JKDXhw_nDCp0ZXErl2PTGkPEBMDqKLTLB8,1608
+tinybird-0.0.1.dev191.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+tinybird-0.0.1.dev191.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
+tinybird-0.0.1.dev191.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
+tinybird-0.0.1.dev191.dist-info/RECORD,,