tinybird 0.0.1.dev189__tar.gz → 0.0.1.dev191__tar.gz

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 0.0.1.dev189
+Version: 0.0.1.dev191
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/prompts.py

@@ -1010,3 +1010,22 @@ Current README.md file:
 <readme>[readme content here]</readme>
 </readme_instructions>
 """
+
+
+def quarantine_prompt(datasource_definition: str) -> str:
+    return f"""
+- You are an expert in Tinybird.
+- You are given a list of rows that went to quarantine during ingestion because of data quality issues.
+- Return the errors in a human readable format so the user can understand what is the problem and fix it.
+- Be concise and to the point.
+- Do not mention clickhouse tables. Refer to data sources instead.
+- The possible fixes recommended fixes are:
+    - Changing a column type in the datasource definition: tell the user what to change in the datasource file, then build again before appending the data.
+    - Changing the data because of the wrong data type: tell the user what to change in the data file, then append the data again.
+- The format of the response always inside the tag <quarantine_errors>[response]</quarantine_errors>
+- Do not use markdown format in the response, because it is a CLI output.
+- The datasource definition is the following:
+<datasource_definition>
+{datasource_definition}
+</datasource_definition>
+"""

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev189'
-__revision__ = 'd55a4dc'
+__version__ = '0.0.1.dev191'
+__revision__ = 'c8535c7'

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/tb/client.py

@@ -1024,11 +1024,7 @@ class TinyB:
         )
 
     async def kafka_list_topics(self, connection_id: str, timeout=5):
-        resp = await self._req(
-            f"/v0/connectors/{connection_id}/preview?preview_activity=false",
-            connect_timeout=timeout,
-            request_timeout=timeout,
-        )
+        resp = await self._req(f"/v0/connectors/{connection_id}/preview?preview_activity=false", timeout=timeout)
         return [x["topic"] for x in resp["preview"]]
 
     async def get_gcp_service_account_details(self) -> Dict[str, Any]:

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/tb/modules/cli.py

@@ -28,7 +28,9 @@ from tinybird.tb.modules.common import (
     CLIException,
     _get_tb_client,
     coro,
+    echo_json,
     echo_safe_format_table,
+    force_echo,
     getenv_bool,
     try_update_config_with_remote,
 )
@@ -72,7 +74,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 @click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
 @click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
 @click.option(
-    "--output", type=click.Choice(["json", "human"], case_sensitive=False), default="human", help="Output format"
+    "--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
 )
 @click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
 @click.version_option(version=VERSION)
@@ -95,7 +97,7 @@ async def cli(
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
     # We need to unpatch for our tests not to break
-    if output == "json":
+    if output != "human":
         __hide_click_output()
     else:
         if show_tokens or not cloud or ctx.invoked_subcommand == "build":
@@ -195,17 +197,10 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
 
 @cli.command()
 @click.argument("query", required=False)
-@click.option("--rows_limit", default=100, help="Max number of rows retrieved")
+@click.option("--rows-limit", default=100, help="Max number of rows retrieved")
 @click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
 @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
 @click.option("--node", default=None, help="The NODE name")
-@click.option(
-    "--format",
-    "format_",
-    type=click.Choice(["json", "csv", "human"], case_sensitive=False),
-    default="human",
-    help="Output format",
-)
 @click.option("--stats/--no-stats", default=False, help="Show query stats")
 @click.pass_context
 @coro
@@ -216,13 +211,13 @@ async def sql(
     pipeline: Optional[str],
     pipe: Optional[str],
     node: Optional[str],
-    format_: str,
     stats: bool,
 ) -> None:
     """Run SQL query over data sources and pipes."""
-
     client = ctx.ensure_object(dict)["client"]
-    req_format = "CSVWithNames" if format_ == "csv" else "JSON"
+    output = ctx.ensure_object(dict)["output"]
+
+    req_format = "CSVWithNames" if output == "csv" else "JSON"
     res = None
     try:
         if query:
@@ -275,11 +270,11 @@ async def sql(
         bytes_read = humanfriendly.format_size(stats_dict["bytes_read"])
         click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
 
-    if format_ == "csv":
-        click.echo(res)
+    if output == "csv":
+        force_echo(str(res))
     elif isinstance(res, dict) and "data" in res and res["data"]:
-        if format_ == "json":
-            click.echo(json.dumps(res, indent=8))
+        if output == "json":
+            echo_json(res, indent=8)
         else:
             dd = []
             for d in res["data"]:

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/tb/modules/common.py

@@ -424,6 +424,14 @@ async def _analyze(filename: str, client: TinyB, format: str, connector: Optiona
     return meta, data
 
 
+async def analyze_file(filename: str, client: TinyB, format: str):
+    meta, data = await _analyze(filename, client, format)
+    schema = meta["analysis"]["schema"]
+    schema = schema.replace(", ", ",\n    ")
+    content = f"""DESCRIPTION >\n    Generated from {filename}\n\nSCHEMA >\n    {schema}"""
+    return content
+
+
 async def _generate_datafile(
     filename: str,
     client: TinyB,
@@ -971,7 +979,7 @@ async def push_data(
     cb.prev_done = 0  # type: ignore[attr-defined]
 
     if not silent:
-        click.echo(FeedbackManager.gray(message=f"\nImporting data to {datasource_name}..."))
+        click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}..."))
 
     if isinstance(url, list):
         urls = url
@@ -1882,15 +1890,19 @@ def get_gcs_connection_name(project_folder) -> str:
     return get_connection_name(project_folder=project_folder, connection_type="GCS")
 
 
-def get_connection_name(project_folder: str, connection_type: str) -> str:
-    connection_name = None
+def get_kafka_connection_name(project_folder: str, connection_name: Optional[str] = None) -> str:
+    return get_connection_name(project_folder=project_folder, connection_type="KAFKA", connection_name=connection_name)
+
+
+def get_connection_name(project_folder: str, connection_type: str, connection_name: Optional[str] = None) -> str:
     valid_pattern = r"^[a-zA-Z][a-zA-Z0-9_]*$"
 
     while not connection_name:
+        short_id = str(uuid.uuid4())[:4]
         connection_name = click.prompt(
-            f"🔗 Enter a name for your new Tinybird {connection_type} connection (use alphanumeric characters, and underscores)",
-            prompt_suffix="\n> ",
+            "Enter a name (only alphanumeric characters and underscores)",
             show_default=True,
+            default=f"{connection_type.lower()}_{short_id}",
         )
         assert isinstance(connection_name, str)
 
@@ -2238,5 +2250,9 @@ def get_error_event(error: str) -> Tuple[str, str]:
     return error_event, silent_error_msg
 
 
-def echo_json(data: Dict[str, Any]) -> None:
-    click.echo(json.dumps(data), force_output=True)  # type: ignore
+def force_echo(string: str) -> None:
+    click.echo(string, force_output=True)  # type: ignore
+
+
+def echo_json(data: Dict[str, Any], indent: Union[None, int, str] = None) -> None:
+    force_echo(json.dumps(data, indent=indent))

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/tb/modules/connection.py

@@ -18,6 +18,7 @@ from tinybird.tb.modules.common import (
     echo_safe_humanfriendly_tables_format_smart_table,
     get_gcs_connection_name,
     get_gcs_svc_account_creds,
+    get_kafka_connection_name,
     get_s3_connection_name,
     production_aws_iamrole_only,
     run_aws_iamrole_connection_flow,
@@ -26,6 +27,7 @@ from tinybird.tb.modules.common import (
 from tinybird.tb.modules.create import (
     generate_aws_iamrole_connection_file_with_secret,
     generate_gcs_connection_file_with_secrets,
+    generate_kafka_connection_with_secrets,
 )
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.project import Project
@@ -275,3 +277,21 @@ async def connection_create_gcs(ctx: Context) -> None:
             connection_path=connection_path,
         )
     )
+
+
+@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
+@click.option("--name", help="The name of the connection")
+@click.pass_context
+@coro
+async def connection_create_kafka(ctx: Context, name: Optional[str] = None) -> None:
+    """
+    Creates a Kafka connection.
+
+    \b
+    $ tb connection create kafka
+    """
+    click.echo(FeedbackManager.highlight(message="» Creating Kafka connection..."))
+    project: Project = ctx.ensure_object(dict)["project"]
+    name = get_kafka_connection_name(project.folder, name)
+    await generate_kafka_connection_with_secrets(name=name, folder=project.folder)
+    click.echo(FeedbackManager.success(message="✓ Done!"))

{tinybird-0.0.1.dev189 → tinybird-0.0.1.dev191}/tinybird/tb/modules/create.py

@@ -409,10 +409,14 @@ def generate_connection_file(name: str, content: str, folder: str, skip_feedback
 
 
 async def generate_aws_iamrole_connection_file_with_secret(
-    name: str, service: str, role_arn_secret_name: str, region: str, folder: str
+    name: str, service: str, role_arn_secret_name: str, region: str, folder: str, with_default_secret: bool = False
 ) -> Path:
+    if with_default_secret:
+        default_secret = ', "arn:aws:iam::123456789012:role/my-role"'
+    else:
+        default_secret = ""
     content = f"""TYPE {service}
-S3_ARN {{{{ tb_secret("{role_arn_secret_name}") }}}}
+S3_ARN {{{{ tb_secret("{role_arn_secret_name}"{default_secret}) }}}}
 S3_REGION {region}
 """
     file_path = generate_connection_file(name, content, folder, skip_feedback=True)
@@ -483,6 +487,7 @@ async def create_resources_from_data(
     data: str,
     project: Project,
     config: Dict[str, Any],
+    skip_pipes: bool = False,
 ) -> List[Path]:
     local_client = await get_tinybird_local_client(config)
     folder_path = project.path
@@ -495,7 +500,7 @@ async def create_resources_from_data(
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -510,7 +515,9 @@ TYPE ENDPOINT
     return result
 
 
-async def create_resources_from_url(url: str, project: Project, config: Dict[str, Any]) -> List[Path]:
+async def create_resources_from_url(
+    url: str, project: Project, config: Dict[str, Any], skip_pipes: bool = False
+) -> List[Path]:
     result: List[Path] = []
     local_client = await get_tinybird_local_client(config)
     format = url.split(".")[-1]
@@ -518,7 +525,7 @@ async def create_resources_from_url(url: str, project: Project, config: Dict[str
     result.append(ds_file)
     name = ds_file.stem
     no_pipes = len(project.get_pipe_files()) == 0
-    if no_pipes:
+    if not skip_pipes and no_pipes:
         pipe_file = generate_pipe_file(
             f"{name}_endpoint",
             f"""
@@ -531,3 +538,14 @@ TYPE ENDPOINT
         )
         result.append(pipe_file)
     return result
+
+
+async def generate_kafka_connection_with_secrets(name: str, folder: str) -> Path:
+    content = """TYPE kafka
+KAFKA_BOOTSTRAP_SERVERS {{ tb_secret("KAFKA_SERVERS", "localhost:9092") }}
+KAFKA_SECURITY_PROTOCOL SASL_SSL
+KAFKA_SASL_MECHANISM PLAIN
+KAFKA_KEY {{ tb_secret("KAFKA_USERNAME", "") }}
+KAFKA_SECRET {{ tb_secret("KAFKA_PASSWORD", "") }}
+"""
+    return generate_connection_file(name, content, folder, skip_feedback=True)