tinybird-cli 5.13.3.dev1__tar.gz → 5.14.0.dev0__tar.gz

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird_cli
-Version: 5.13.3.dev1
+Version: 5.14.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -18,14 +18,25 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.13.3.dev1
+5.14.0.dev0
+***********
+
+- `Added` support for `KAFKA_KEY_FORMAT` and `KAFKA_VALUE_FORMAT` parameters for kafka datasources.
+
+5.13.3
 ***********
 
 - `Fixed` updating the cron schedule of a BigQuery data source when having a query but no external_data_source setting.
+- `Fixed` handle Kafka connector with multiple brokers properly.
 
 5.13.2
 ***********
 
+- `Fixed` Correctly handle Kafka connector with multiple brokers
+
+5.13.1.dev2
+***********
+
 - `Added` comply with [PEP 625](https://peps.python.org/pep-0625/)
 - `Added` `eu-west-1` region to the list of available regions.
 

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.13.3.dev1'
-__revision__ = '3be7e20'
+__version__ = '5.14.0.dev0'
+__revision__ = '643b049'

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/client.py

@@ -72,7 +72,7 @@ def parse_error_response(response: Response) -> str:
         if content.get("error", None):
             error = content["error"]
             if content.get("errors", None):
-                error += f' -> errors: {content.get("errors")}'
+                error += f" -> errors: {content.get('errors')}"
         else:
             error = json.dumps(response, indent=4)
         return error

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/connectors.py

@@ -246,7 +246,7 @@ class BigQuery(Connector):
                     uri='{self.gcs.gs_url()}{destination}*.csv',
                     format='CSV',
                     overwrite=true,
-                    header={ "true" if with_headers else "false" },
+                    header={"true" if with_headers else "false"},
                     field_delimiter=',') AS
                     {sql}
                 """
@@ -319,7 +319,7 @@ class Snowflake(Connector):
 
     def create_stage(self):
         sql = f"""
-                    create stage "{self.options['schema']}".{self.stage()}
+                    create stage "{self.options["schema"]}".{self.stage()}
                         url='{self.gcs.gcs_url()}'
                         storage_integration = {self.storage_integration()};
                 """
@@ -337,7 +337,7 @@ class Snowflake(Connector):
                     from ({sql})
                     overwrite = true
                     file_format = (TYPE=CSV COMPRESSION=NONE ESCAPE_UNENCLOSED_FIELD=NONE FIELD_DELIMITER='|' FIELD_OPTIONALLY_ENCLOSED_BY='"' null_if=())
-                    header = {"true" if with_headers else "false" }
+                    header = {"true" if with_headers else "false"}
                     max_file_size = 2500000000;
                 """
         self.execute(sql)

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/datafile.py

@@ -1218,6 +1218,8 @@ def parse(
         "kafka_store_raw_value": assign_var("kafka_store_raw_value"),
         "kafka_store_headers": assign_var("kafka_store_headers"),
         "kafka_store_binary_headers": assign_var("kafka_store_binary_headers"),
+        "kafka_key_format": assign_var("kafka_key_format"),
+        "kafka_value_format": assign_var("kafka_value_format"),
         "kafka_key_avro_deserialization": assign_var("kafka_key_avro_deserialization"),
         "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
         "kafka_sasl_mechanism": assign_var("kafka_sasl_mechanism"),
@@ -2175,8 +2177,8 @@ class PipeCheckerRunner:
                             AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                             AND error = 0
                             AND not mapContains(parameters, '__tb__semver')
-                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
-                            { extra_where_clause }
+                            {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
+                            {extra_where_clause}
                         Limit 5000000 -- Enough to bring data while not processing all requests from highly used pipes
                         )
                     group by request_param_names, http_method
@@ -2202,7 +2204,7 @@ class PipeCheckerRunner:
                                         AND extractURLParameter(assumeNotNull(url), 'debug') <> 'query'
                                         AND error = 0
                                         AND not mapContains(parameters, '__tb__semver')
-                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ''}
+                                        {" AND " + " AND ".join([f"mapContains(pipe_request_params, '{match}')" for match in matches]) if matches and len(matches) > 0 else ""}
                                         {extra_where_clause}
                                     LIMIT {limit}
                                 )
@@ -4425,7 +4427,7 @@ async def folder_push(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -4442,7 +4444,7 @@ async def folder_push(
                     if raise_on_exists:
                         raise AlreadyExistsException(
                             FeedbackManager.warning_name_already_exists(
-                                name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
+                                name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
                             )
                         )
                     else:
@@ -4457,7 +4459,7 @@ async def folder_push(
                                     name=(
                                         name
                                         if to_run[name]["version"] is None
-                                        else f'{name}__v{to_run[name]["version"]}'
+                                        else f"{name}__v{to_run[name]['version']}"
                                     )
                                 )
                             )
@@ -5013,7 +5015,7 @@ async def format_engine(
     else:
         if node.get("engine", None):
             empty = '""'
-            file_parts.append(f'ENGINE {node["engine"]["type"]}' if node.get("engine", {}).get("type") else empty)
+            file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
             file_parts.append(DATAFILE_NEW_LINE)
             for arg in sorted(node["engine"].get("args", [])):
                 elem = ", ".join([x.strip() for x in arg[1].split(",")])
@@ -5030,7 +5032,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
     if node_type == PipeNodeTypes.MATERIALIZED:
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
-        file_parts.append(f'DATASOURCE {node["datasource"]}')
+        file_parts.append(f"DATASOURCE {node['datasource']}")
         file_parts.append(DATAFILE_NEW_LINE)
         await format_engine(file_parts, node)
 
@@ -5038,10 +5040,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
     if node_type == PipeNodeTypes.COPY:
         file_parts.append(node_type_upper)
         file_parts.append(DATAFILE_NEW_LINE)
-        file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
+        file_parts.append(f"TARGET_DATASOURCE {node['target_datasource']}")
         if node.get("mode"):
             file_parts.append(DATAFILE_NEW_LINE)
-            file_parts.append(f'COPY_MODE {node.get("mode")}')
+            file_parts.append(f"COPY_MODE {node.get('mode')}")
 
         if node.get(CopyParameters.COPY_SCHEDULE):
             is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
@@ -5095,7 +5097,7 @@ async def format_node(
     if item and not unroll_includes:
         return
 
-    file_parts.append(f'NODE {node["name"].strip()}')
+    file_parts.append(f"NODE {node['name'].strip()}")
     file_parts.append(DATAFILE_NEW_LINE)
 
     from collections import namedtuple

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/feedback_manager.py

@@ -951,7 +951,7 @@ Ready? """
     )
     success_datasource_alter = success_message("** The Data Source has been correctly updated.")
     success_datasource_kafka_connected = success_message(
-        "** Data Source '{id}' created\n" "** Kafka streaming connection configured successfully!"
+        "** Data Source '{id}' created\n** Kafka streaming connection configured successfully!"
     )
     success_datasource_shared = success_message(
         "** The Data Source {datasource} has been correctly shared with {workspace}"

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/sql.py

@@ -86,7 +86,7 @@ def get_format(sql: str) -> Optional[str]:
     """
     FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.findall(FORMAT_RE, sql, re.I)
+    format = re.findall(FORMAT_RE, sql, re.IGNORECASE)
     return format[0] if format else None
 
 
@@ -100,7 +100,7 @@ def get_format_group(sql: str) -> str:
     """
     FORMAT_RE = r"\s+format\s+(\w+)\s*$"
     sql = sql.strip()
-    format = re.search(FORMAT_RE, sql, re.I)
+    format = re.search(FORMAT_RE, sql, re.IGNORECASE)
     return format.group() if format else ""
 
 
@@ -135,7 +135,7 @@ def remove_format(sql: str) -> str:
     """
     FORMAT_RE = r"\s+(format)\s+(\w+)\s*$"
     sql = sql.strip()
-    return re.sub(FORMAT_RE, "", sql, flags=re.I)
+    return re.sub(FORMAT_RE, "", sql, flags=re.IGNORECASE)
 
 
 def col_name(name: str, backquotes: bool = True) -> str:
@@ -241,7 +241,7 @@ def format_parse_error(
         message += f" found at position {adjusted_position - len(keyword)}"
     else:
         message += (
-            f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
+            f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
         )
     return message
 

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/sql_template_fmt.py

@@ -100,7 +100,7 @@ def _format_jinja_node(self, node: Node, max_length: int) -> bool:
                 parts = tag.code.split("\n")
                 prefix = INDENT * (node.depth[0] + node.depth[1])
                 if len(parts) > 1:
-                    tag.code = "\n".join([f'{prefix if i != 0 else ""}{part}' for i, part in enumerate(parts)])
+                    tag.code = "\n".join([f"{prefix if i != 0 else ''}{part}" for i, part in enumerate(parts)])
 
         node.value = str(tag)
 

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/sql_toolset.py

@@ -3,7 +3,7 @@ import logging
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
-from typing import Any, FrozenSet, List, Optional, Set, Tuple
+from typing import FrozenSet, List, Optional, Set, Tuple
 
 from chtoolset import query as chquery
 from toposort import toposort
@@ -172,7 +172,7 @@ def tables_or_sql(replacement: dict, table_functions=False) -> set:
         return {replacement}
 
 
-def _separate_as_tuple_if_contains_database_and_table(definition: str) -> Any:
+def _separate_as_tuple_if_contains_database_and_table(definition: str) -> str | Tuple[str, str]:
     if "." in definition:
         database_and_table_separated = definition.split(".")
         return database_and_table_separated[0], database_and_table_separated[1]
@@ -219,6 +219,7 @@ def replace_tables(
     output_one_line: bool = False,
     timestamp: Optional[datetime] = None,
     function_allow_list: Optional[FrozenSet[str]] = None,
+    original_replacements: Optional[dict] = None,
 ) -> str:
     """
     Given a query and a list of table replacements, returns the query after applying the table replacements.
@@ -239,6 +240,12 @@ def replace_tables(
         _replacements[rk] = r if isinstance(r, tuple) else (default_database, r)
         _replaced_with.add(r)
 
+    if original_replacements:
+        # Some replacements have been expanded by filters and turned to a query str, but we need to send the original
+        # ones to is_invalid_resource()
+        for r in original_replacements.values():
+            _replaced_with.add(r)
+
     deps: defaultdict = defaultdict(set)
     _tables = sql_get_used_tables(
         sql,

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/tb_cli_modules/cli.py

@@ -1053,7 +1053,7 @@ async def materialize(
         return node, node_name
 
     def _choose_target_datasource_name(pipe, node, node_name):
-        return target_datasource or node.get("datasource", None) or f'mv_{pipe["resource_name"]}_{node_name}'
+        return target_datasource or node.get("datasource", None) or f"mv_{pipe['resource_name']}_{node_name}"
 
     def _save_local_backup_pipe(pipe):
         pipe_bak = f"{filename}_bak"
@@ -1231,7 +1231,7 @@ def __patch_click_output():
             if isinstance(substitution, str):
                 msg = re.sub(pattern, substitution, str(msg))
             else:
-                msg = re.sub(pattern, lambda m: substitution(m.group(0)), str(msg))  # noqa
+                msg = re.sub(pattern, lambda m: substitution(m.group(0)), str(msg))  # noqa: B023
         return msg
 
     def _obfuscate_echo(msg: Any, *args: Any, **kwargs: Any) -> None:

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/tb_cli_modules/common.py

@@ -1252,26 +1252,28 @@ def is_url_valid(url):
         return False
 
 
-def validate_kafka_bootstrap_servers(host_and_port):
-    if not isinstance(host_and_port, str):
+def validate_kafka_bootstrap_servers(bootstrap_servers):
+    if not isinstance(bootstrap_servers, str):
         raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
-    parts = host_and_port.split(":")
-    if len(parts) > 2:
-        raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
-    host = parts[0]
-    port_str = parts[1] if len(parts) == 2 else "9092"
-    try:
-        port = int(port_str)
-    except Exception:
-        raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
-    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+
+    for host_and_port in bootstrap_servers.split(","):
+        parts = host_and_port.split(":")
+        if len(parts) > 2:
+            raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
+        host = parts[0]
+        port_str = parts[1] if len(parts) == 2 else "9092"
         try:
-            sock.settimeout(3)
-            sock.connect((host, port))
-        except socket.timeout:
-            raise CLIException(FeedbackManager.error_kafka_bootstrap_server_conn_timeout())
+            port = int(port_str)
         except Exception:
-            raise CLIException(FeedbackManager.error_kafka_bootstrap_server_conn())
+            raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
+        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+            try:
+                sock.settimeout(3)
+                sock.connect((host, port))
+            except socket.timeout:
+                raise CLIException(FeedbackManager.error_kafka_bootstrap_server_conn_timeout())
+            except Exception:
+                raise CLIException(FeedbackManager.error_kafka_bootstrap_server_conn())
 
 
 def validate_kafka_key(s):
@@ -1588,7 +1590,7 @@ async def try_update_config_with_remote(
 def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
     return (
         click.prompt(
-            f"\nCopy the \"admin your@email\" token from {ui_host}/tokens and paste it here { 'OR press enter to use the token from .tinyb file' if default_token else ''}",
+            f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
             hide_input=True,
             show_default=False,
             default=default_token,

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/tb_cli_modules/pipe.py

@@ -117,7 +117,7 @@ async def pipe_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: str):
             sumIf(error_count, date > now() - interval 7 day) errors,
             avgMergeIf(avg_duration_state, date > now() - interval 7 day) latency
         FROM tinybird.pipe_stats
-        WHERE pipe_id in ({','.join(pipes_to_get_stats)})
+        WHERE pipe_id in ({",".join(pipes_to_get_stats)})
         GROUP BY pipe_id
         ORDER BY requests DESC
         FORMAT JSON

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py

@@ -155,7 +155,7 @@ def parse_file(file: str) -> Iterable[TestCase]:
                 )
         except Exception as e:
             raise CLIException(
-                f"""Error: {FeedbackManager.error_exception(error=e)} reading file, check "{file}"->"{definition.get('name')}" """
+                f"""Error: {FeedbackManager.error_exception(error=e)} reading file, check "{file}"->"{definition.get("name")}" """
             )
 
 

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird/tornado_template.py

@@ -370,7 +370,7 @@ class Template:
         for chunk in self.file.body.chunks:
             if isinstance(chunk, _ExtendsBlock):
                 if not loader:
-                    raise ParseError("{% extends %} block found, but no " "template loader")
+                    raise ParseError("{% extends %} block found, but no template loader")
                 template = loader.load(chunk.name, self.name)
                 ancestors.extend(template._get_ancestors(loader))
         return ancestors
@@ -633,7 +633,7 @@ class _Expression(_Node):
 
     def generate(self, writer):
         writer.write_line("_tt_tmp = %s" % self.expression, self.line)
-        writer.write_line("if isinstance(_tt_tmp, _tt_string_types):" " _tt_tmp = _tt_utf8(_tt_tmp)", self.line)
+        writer.write_line("if isinstance(_tt_tmp, _tt_string_types): _tt_tmp = _tt_utf8(_tt_tmp)", self.line)
         writer.write_line("else: _tt_tmp = _tt_utf8(str(_tt_tmp))", self.line)
         if not self.raw and writer.current_template.autoescape is not None:
             # In python3 functions like xhtml_escape return unicode,

{tinybird_cli-5.13.3.dev1 → tinybird_cli-5.14.0.dev0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.13.3.dev1
+Version: 5.14.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -18,14 +18,25 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.13.3.dev1
+5.14.0.dev0
+***********
+
+- `Added` support for `KAFKA_KEY_FORMAT` and `KAFKA_VALUE_FORMAT` parameters for kafka datasources.
+
+5.13.3
 ***********
 
 - `Fixed` updating the cron schedule of a BigQuery data source when having a query but no external_data_source setting.
+- `Fixed` handle Kafka connector with multiple brokers properly.
 
 5.13.2
 ***********
 
+- `Fixed` Correctly handle Kafka connector with multiple brokers
+
+5.13.1.dev2
+***********
+
 - `Added` comply with [PEP 625](https://peps.python.org/pep-0625/)
 - `Added` `eu-west-1` region to the list of available regions.