tinybird-cli 5.9.1.dev2__tar.gz → 5.10.0__tar.gz

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev2
+Version: 5.10.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,11 +18,13 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.9.1.dev2
+5.10.0
 ***********
 
-- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
 - `Added` new "File not found" error to `tb check` when including files from missing paths.
+- `Added` support for Kafka Data Sources with CA certificate.
+- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Fixed` Correctly parse lambda expressions in indexes
 
 5.9.0
 ***********

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.9.1.dev2'
-__revision__ = '4f458c3'
+__version__ = '5.10.0'
+__revision__ = 'f8aa90b'

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/ch_utils/engine.py

@@ -4,7 +4,14 @@ from collections import defaultdict
 from dataclasses import asdict
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
-from ..sql import TableIndex, col_name, engine_replicated_to_local, parse_indexes_structure, parse_table_structure
+from ..sql import (
+    TableIndex,
+    TableProjection,
+    col_name,
+    engine_replicated_to_local,
+    parse_indexes_structure,
+    parse_table_structure,
+)
 
 DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
 DEFAULT_JOIN_EMPTY_PARAMETERS = ["join_strictness", "join_type", "key_columns"]
@@ -237,6 +244,10 @@ class TableDetails:
     def indexes(self) -> List[TableIndex]:
         return _parse_indexes(str(self.details.get("create_table_query", "")))
 
+    @property
+    def projections(self) -> List[TableProjection]:
+        return _parse_projections(self.details.get("create_table_query", ""))
+
     def to_json(self, exclude: Optional[List[str]] = None, include_empty_details: bool = False):
         # name, database are not exported since they are not part of the engine
         d: Dict[str, Any] = {
@@ -833,3 +844,12 @@ def _parse_indexes(create_table_query_expr: str) -> List[TableIndex]:
         return []
 
     return parse_indexes_structure(indexes)
+
+
+def _parse_projections(create_table_query_expr: str) -> List[TableProjection]:
+    return [
+        TableProjection(name, expr)
+        for name, expr in re.findall(
+            r"PROJECTION\s+(\w+)\s*\(((?:[^()]|\((?:[^()]|\([^()]*\))*\))*)\)", create_table_query_expr
+        )
+    ]

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/client.py

@@ -111,7 +111,7 @@ class TinyB(object):
         retries: int = LIMIT_RETRIES,
         use_token: Optional[str] = None,
         **kwargs,
-    ):  # noqa: C901
+    ):
         url = f"{self.host.strip('/')}/{endpoint.strip('/')}"
 
         token_to_use = use_token if use_token else self.token
@@ -275,7 +275,7 @@ class TinyB(object):
             bigquery_connection = (
                 await self.bigquery_connection() if connector == "bigquery" or connector is None else None
             )
-        connectors = connectors + [bigquery_connection] if bigquery_connection else connectors
+        connectors = [*connectors, bigquery_connection] if bigquery_connection else connectors
         if connector:
             return [
                 {
@@ -865,6 +865,7 @@ class TinyB(object):
         kafka_auto_offset_reset=None,
         kafka_schema_registry_url=None,
         kafka_sasl_mechanism="PLAIN",
+        kafka_ssl_ca_pem=None,
     ):
         params = {
             "service": "kafka",
@@ -880,7 +881,8 @@ class TinyB(object):
             params["kafka_schema_registry_url"] = kafka_schema_registry_url
         if kafka_auto_offset_reset:
             params["kafka_auto_offset_reset"] = kafka_auto_offset_reset
-
+        if kafka_ssl_ca_pem:
+            params["kafka_ssl_ca_pem"] = kafka_ssl_ca_pem
         connection_params = {key: value for key, value in params.items() if value is not None}
 
         return await self._req(

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/datafile.py

@@ -60,7 +60,13 @@ from toposort import toposort
 from tinybird.config import PROJECT_PATHS
 from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH, format_sql_template
 from tinybird.syncasync import sync_to_async
-from tinybird.tb_cli_modules.common import _get_tb_client, get_current_main_workspace, getenv_bool, wait_job
+from tinybird.tb_cli_modules.common import (
+    _get_tb_client,
+    get_ca_pem_content,
+    get_current_main_workspace,
+    getenv_bool,
+    wait_job,
+)
 from tinybird.tb_cli_modules.config import CLIConfig
 from tinybird.tb_cli_modules.exceptions import CLIGitReleaseException, CLIPipeException
 
@@ -72,7 +78,7 @@ from .sql_template import get_template_and_variables, get_used_tables_in_templat
 from .tornado_template import UnClosedIfError
 
 os.environ["GIT_PYTHON_REFRESH"] = "quiet"
-from git import HEAD, Diff, GitCommandError, InvalidGitRepositoryError, Repo  # noqa: E402
+from git import HEAD, Diff, GitCommandError, InvalidGitRepositoryError, Repo
 
 INTERNAL_TABLES: Tuple[str, ...] = (
     "datasources_ops_log",
@@ -586,7 +592,7 @@ class Deployment:
                 self.cli_git_release.validate_local_for_release(self.current_release, check_outdated=check_outdated)
             click.echo(FeedbackManager.info_deployment_detecting_changes_header())
             commit = self.cli_git_release.get_main_branch_commit() if use_main else self.current_release["commit"]
-            diffs = self.cli_git_release.diff_datafiles(commit)
+            diffs = self.cli_git_release.diff_datafiles(commit)  # type: ignore
             click.echo(
                 FeedbackManager.info_git_release_diffs(
                     workspace=self.current_ws["name"],
@@ -960,7 +966,7 @@ def parse(
     basepath: str = ".",
     replace_includes: bool = True,
     skip_eval: bool = False,
-) -> Datafile:  # noqa: C901
+) -> Datafile:
     """
     Parses `s` string into a document
     >>> d = parse("FROM SCRATCH\\nSOURCE 'https://example.com'\\n#this is a comment\\nMAINTAINER 'rambo' #this is me\\nNODE \\"test_01\\"\\n    DESCRIPTION this is a node that does whatever\\nSQL >\\n\\n        SELECT * from test_00\\n\\n\\nNODE \\"test_02\\"\\n    DESCRIPTION this is a node that does whatever\\nSQL >\\n\\n    SELECT * from test_01\\n    WHERE a > 1\\n    GROUP by a\\n")
@@ -1110,9 +1116,10 @@ def parse(
                     pass
                 finally:
                     file.seek(0)
-                lines[lineno : lineno + 1] = [""] + list(
-                    StringIO(Template(file.read()).safe_substitute(attrs), newline=None)
-                )
+                lines[lineno : lineno + 1] = [
+                    "",
+                    *list(StringIO(Template(file.read()).safe_substitute(attrs), newline=None)),
+                ]
         except FileNotFoundError:
             raise IncludeFileNotFoundException(f, lineno)
 
@@ -1211,6 +1218,7 @@ def parse(
         "kafka_store_headers": assign_var("kafka_store_headers"),
         "kafka_store_binary_headers": assign_var("kafka_store_binary_headers"),
         "kafka_key_avro_deserialization": assign_var("kafka_key_avro_deserialization"),
+        "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
         "import_service": assign_var("import_service"),
         "import_connection_name": assign_var("import_connection_name"),
         "import_schedule": assign_var("import_schedule"),
@@ -1318,7 +1326,7 @@ async def process_file(
     workspace_map: Optional[Dict] = None,
     workspace_lib_paths: Optional[List[Tuple[str, str]]] = None,
     current_ws: Optional[Dict[str, Any]] = None,
-):  # noqa: C901 B006
+):
     if workspace_map is None:
         workspace_map = {}
 
@@ -1349,6 +1357,7 @@ async def process_file(
                     "kafka_connection_name": params.get("kafka_connection_name", None),
                     "kafka_auto_offset_reset": params.get("kafka_auto_offset_reset", None),
                     "kafka_schema_registry_url": params.get("kafka_schema_registry_url", None),
+                    "kafka_ssl_ca_pem": get_ca_pem_content(params.get("kafka_ssl_ca_pem", None), filename),
                 }
 
                 connector = await tb_client.get_connection(**connector_params)
@@ -2118,7 +2127,7 @@ class PipeCheckerRunnerResponse:
 
 
 class PipeCheckerRunner:
-    checker_stream_result_class = unittest.runner._WritelnDecorator  # type: ignore
+    checker_stream_result_class = unittest.runner._WritelnDecorator
 
     def __init__(self, pipe_name: str, host: str):
         self.pipe_name = pipe_name
@@ -2281,7 +2290,7 @@ class PipeCheckerRunner:
             )
 
         result = PipeCheckerTextTestResult(
-            self.checker_stream_result_class(sys.stdout), descriptions=True, verbosity=2, custom_output=custom_output
+            self.checker_stream_result_class(sys.stdout), descriptions=True, verbosity=2, custom_output=custom_output  # type: ignore
         )
         result.failfast = failfast
         suite.run(result)
@@ -2868,7 +2877,7 @@ async def new_pipe(
     config: Any = None,
     fork_downstream: Optional[bool] = False,
     fork: Optional[bool] = False,
-):  # noqa: C901
+):
     # TODO use tb_client instead of calling the urls directly.
     host = tb_client.host
     token = tb_client.token
@@ -3270,10 +3279,10 @@ async def new_ds(
                     if job_url:
                         click.echo(FeedbackManager.success_dynamodb_initial_load(job_url=job_url))
 
-            if "tokens" in ds and ds["tokens"]:
+            if ds.get("tokens"):
                 await manage_tokens()
 
-            if "shared_with" in ds and ds["shared_with"]:
+            if ds.get("shared_with"):
                 if not user_token:
                     click.echo(FeedbackManager.info_skipping_shared_with_entry())
                 else:
@@ -3989,7 +3998,7 @@ async def build_graph(
                     mapped_workspaces.append(
                         workspace_map.get(shared_with)
                         if workspace_map.get(shared_with, None) is not None
-                        else shared_with
+                        else shared_with  # type: ignore
                     )
                 r["shared_with"] = mapped_workspaces
 
@@ -4189,7 +4198,7 @@ async def folder_push(
     use_main: bool = False,
     check_outdated: bool = True,
     hide_folders: bool = False,
-):  # noqa: C901
+):
     workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
     current_ws: Dict[str, Any] = next(
         (workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
@@ -4754,7 +4763,7 @@ DATAFILE_INDENT = " " * 4
 
 
 def format_schema(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
-    if "schema" in node and node["schema"]:
+    if node.get("schema"):
         file_parts.append("SCHEMA >")
         file_parts.append(DATAFILE_NEW_LINE)
         columns = schema_to_sql_columns(node["columns"])
@@ -4766,7 +4775,7 @@ def format_schema(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
 
 
 def format_indices(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
-    if "indexes" in node and node["indexes"]:
+    if node.get("indexes"):
         indexes = node["indexes"]
         file_parts.append("INDEXES >")
         file_parts.append(DATAFILE_NEW_LINE)
@@ -4995,7 +5004,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
             file_parts.append(DATAFILE_NEW_LINE)
             file_parts.append(f'COPY_MODE {node.get("mode")}')
 
-        if CopyParameters.COPY_SCHEDULE in node and node[CopyParameters.COPY_SCHEDULE]:
+        if node.get(CopyParameters.COPY_SCHEDULE):
             is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
             file_parts.append(DATAFILE_NEW_LINE)
             file_parts.append(
@@ -5154,7 +5163,7 @@ async def folder_pull(
     verbose: bool = True,
     progress_bar: bool = False,
     fmt: bool = False,
-):  # noqa: C901
+):
     pattern = re.compile(match) if match else None
 
     def _get_latest_versions(resources: List[str]):
@@ -5321,7 +5330,7 @@ async def diff_command(
 
     if filenames:
         if len(filenames) == 1:
-            filenames = [filenames[0]] + get_project_filenames(filenames[0])
+            filenames = [filenames[0], *get_project_filenames(filenames[0])]
         await folder_pull(client, target_dir, False, None, True, verbose=False)
     else:
         filenames = get_project_filenames(".")

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/feedback_manager.py

@@ -238,6 +238,8 @@ class FeedbackManager:
     error_connection_does_not_exists = error_message("Connection {connection_id} does not exist")
     error_connection_create = error_message("Connection {connection_name} could not be created: {error}")
     error_connection_integration_not_available = error_message("Connection could not be created: {error}")
+    error_connection_invalid_ca_pem = error_message("Invalid CA certificate in PEM format")
+    error_connection_ca_pem_not_found = error_message("CA certificate in PEM format not found at {ca_pem}")
     error_workspace = error_message("Workspace {workspace} not found. use 'tb workspace ls' to list your workspaces")
     error_deleted_include = error_message(
         "Related include file {include_file} was deleted and it's used in {filename}. Delete or remove dependency from {filename}."

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/sql.py

@@ -38,11 +38,37 @@ class TableIndex:
         return f"CLEAR INDEX IF EXISTS {self.name}"
 
 
+@dataclass
+class TableProjection:
+    """Defines a CH table PROJECTION"""
+
+    name: str
+    expr: str
+
+    def to_datafile(self):
+        return f"{self.name} ({self.expr})"
+
+    def to_sql(self):
+        return f"PROJECTION {self.to_datafile()}"
+
+    def add_index_sql(self):
+        return f"ADD {self.to_sql()}"
+
+    def drop_index_sql(self):
+        return f"DROP PROJECTION IF EXISTS {self.name}"
+
+    def materialize_index_sql(self):
+        return f"MATERIALIZE PROJECTION IF EXISTS {self.name}"
+
+    def clear_index_sql(self):
+        return f"CLEAR PROJECTION IF EXISTS {self.name}"
+
+
 def as_subquery(sql: str) -> str:
     return f"""(\n{sql}\n)"""
 
 
-def get_format(sql: str) -> str:
+def get_format(sql: str) -> Optional[str]:
     """
     retrieves FORMAT from CH sql
     >>> get_format('select * from test')
@@ -211,7 +237,7 @@ def format_parse_error(
     return message
 
 
-def parse_indexes_structure(indexes: List[str]) -> List[TableIndex]:
+def parse_indexes_structure(indexes: Optional[List[str]]) -> List[TableIndex]:
     """
     >>> parse_indexes_structure(["index_name a TYPE set(100) GRANULARITY 100", "index_name_bf mapValues(d) TYPE bloom_filter(0.001) GRANULARITY 16"])
     [TableIndex(name='index_name', expr='a', type_full='set(100)', granularity='100'), TableIndex(name='index_name_bf', expr='mapValues(d)', type_full='bloom_filter(0.001)', granularity='16')]
@@ -239,6 +265,8 @@ def parse_indexes_structure(indexes: List[str]) -> List[TableIndex]:
     ValueError: invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`
     >>> parse_indexes_structure(["my_index m['key'] TYPE ngrambf_v1(1, 1024, 1, 42) GRANULARITY 1"])
     [TableIndex(name='my_index', expr="m['key']", type_full='ngrambf_v1(1, 1024, 1, 42)', granularity='1')]
+    >>> parse_indexes_structure(["my_index_lambda arrayMap(x -> tupleElement(x,'message'), column_name) TYPE ngrambf_v1(1, 1024, 1, 42) GRANULARITY 1"])
+    [TableIndex(name='my_index_lambda', expr="arrayMap(x -> tupleElement(x,'message'), column_name)", type_full='ngrambf_v1(1, 1024, 1, 42)', granularity='1')]
     >>> parse_indexes_structure(["ip_range_minmax_idx (toIPv6(ip_range_start), toIPv6(ip_range_end)) TYPE minmax GRANULARITY 1"])
     [TableIndex(name='ip_range_minmax_idx', expr='(toIPv6(ip_range_start), toIPv6(ip_range_end))', type_full='minmax', granularity='1')]
     """
@@ -253,7 +281,7 @@ def parse_indexes_structure(indexes: List[str]) -> List[TableIndex]:
             raise ValueError("invalid INDEX format. Usage: `name expr TYPE type_full GRANULARITY granularity`")
 
         match = re.match(
-            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\".]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
+            r"(\w+)\s+([\w\s*\[\]\*\(\),\'\"-><.]+)\s+TYPE\s+(\w+)(?:\(([\w\s*.,]+)\))?(?:\s+GRANULARITY\s+(\d+))?",
             index,
         )
         if match:
@@ -498,7 +526,7 @@ REGEX_WHITESPACE = re.compile(r"\s*")
 REGEX_COMMENT = re.compile(r"\-\-[^\n\r]*[\n\r]")
 
 
-def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
+def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
     # CH syntax from https://clickhouse.com/docs/en/sql-reference/statements/create/table/
     # name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1]
     try:

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/sql_template.py

@@ -359,7 +359,7 @@ def defined(x=None):
     return True
 
 
-def array_type(types):  # noqa: C901
+def array_type(types):
     def _f(
         x, _type=None, default=None, defined=True, required=None, description=None, enum=None, example=None, format=None
     ):
@@ -1370,7 +1370,7 @@ _namespace = {
 }
 
 
-reserved_vars = set(["_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error"] + list(vars(builtins)))
+reserved_vars = set(["_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))])
 for p in DEFAULT_PARAM_NAMES:  # we handle these in an specific manner
     reserved_vars.discard(p)  # `format` is part of builtins
 error_vars = ["error", "custom_error"]
@@ -1552,7 +1552,7 @@ def get_var_names(t):
         raise SQLTemplateException(e)
 
 
-def get_var_data(content, node_id=None):  # noqa: C901
+def get_var_data(content, node_id=None):
     def node_to_value(x):
         if type(x) in (ast.Bytes, ast.Str):
             return x.s
@@ -1619,8 +1619,10 @@ def get_var_data(content, node_id=None):  # noqa: C901
 
                     kwargs = {}
                     for x in node.keywords:
-                        kwargs[x.arg] = node_to_value(x.value)
-
+                        value = node_to_value(x.value)
+                        kwargs[x.arg] = value
+                        if x.arg == "default":
+                            kwargs["default"] = check_default_value(value)
                     if func in VALID_CUSTOM_FUNCTION_NAMES:
                         # Type definition here is set to 'String' because it comes from a
                         # `defined(variable)` expression that does not contain any type hint.
@@ -1628,11 +1630,15 @@ def get_var_data(content, node_id=None):  # noqa: C901
                         # args[0] check is used to avoid adding unnamed parameters found in
                         # templates like: `split_to_array('')`
                         if len(args) > 0 and args[0] not in vars and args[0]:
-                            vars[args[0]] = {"type": "String", "default": None, "used_in": "function_call"}
+                            vars[args[0]] = {
+                                "type": "String",
+                                "default": None,
+                                "used_in": "function_call",
+                            }
                     elif func == "Array":
                         if "default" not in kwargs:
                             default = kwargs.get("default", args[2] if len(args) > 2 and args[2] else None)
-                            kwargs["default"] = default
+                            kwargs["default"] = check_default_value(default)
                         if len(args):
                             vars[args[0]] = {
                                 "type": f"Array({args[1]})" if len(args) > 1 else "Array(String)",
@@ -1644,9 +1650,11 @@ def get_var_data(content, node_id=None):  # noqa: C901
                             # if this is a cast use the function name to get the type
                             if "default" not in kwargs:
                                 default = kwargs.get("default", args[1] if len(args) > 1 else None)
-                                kwargs["default"] = default
+                                kwargs["default"] = check_default_value(default)
                             try:
                                 vars[args[0]] = {"type": func, **kwargs}
+                                if "default" in kwargs:
+                                    kwargs["default"] = check_default_value(kwargs["default"])
                             except TypeError as e:
                                 logging.exception(f"pipe parsing problem {content} (node '{node_id}'):  {e}")
                 except Exception as e:
@@ -1665,6 +1673,13 @@ def get_var_data(content, node_id=None):  # noqa: C901
 
         return vars
 
+    def check_default_value(value):
+        if isinstance(value, int):
+            MAX_SAFE_INTEGER = 9007199254740991
+            if value > MAX_SAFE_INTEGER:
+                return str(value)
+        return value
+
     def parse_content(content, retries=0):
         try:
             parsed = ast.parse(content)
@@ -1688,7 +1703,7 @@ def get_var_data(content, node_id=None):  # noqa: C901
     return [dict(name=k, **v) for k, v in vars.items()]
 
 
-def get_var_names_and_types(t, node_id=None):  # noqa: C901
+def get_var_names_and_types(t, node_id=None):
     """
     >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE description = {{Float32(with_value, 0.0)}}"))
     [{'name': 'with_value', 'type': 'Float32', 'default': 0.0}]
@@ -1744,6 +1759,8 @@ def get_var_names_and_types(t, node_id=None):  # noqa: C901
     [{'name': 'symbol_id', 'type': 'Int128', 'description': 'Symbol Id', 'required': True, 'default': 11111}, {'name': 'user_id', 'type': 'Int256', 'description': 'User Id', 'default': 3555}]
     >>> get_var_names_and_types(Template("SELECT now() > {{DateTime64(timestamp, '2020-09-09 10:10:10.000')}}"))
     [{'name': 'timestamp', 'type': 'DateTime64', 'default': '2020-09-09 10:10:10.000'}]
+    >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE symbol = {{Int64(symbol_id, 9223372036854775807)}}"))
+    [{'name': 'symbol_id', 'type': 'Int64', 'default': '9223372036854775807'}]
     """
     try:
 

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/syncasync.py

@@ -177,7 +177,7 @@ class _WorkItem:
             return
         try:
             result = self.fn(*self.args, **self.kwargs)
-        except BaseException as exc:  # noqa: B036
+        except BaseException as exc:
             self.future.set_exception(exc)
             # Break a reference cycle with the exception 'exc'
             self = None
@@ -473,11 +473,11 @@ class AsyncToSync:
             if exc_info[1]:
                 try:
                     raise exc_info[1]
-                except BaseException:  # noqa: B036
+                except BaseException:
                     result = await self.awaitable(*args, **kwargs)
             else:
                 result = await self.awaitable(*args, **kwargs)
-        except BaseException as e:  # noqa: B036
+        except BaseException as e:
             call_result.set_exception(e)
         else:
             call_result.set_result(result)
@@ -639,7 +639,7 @@ class SyncToAsync:
             if exc_info[1]:
                 try:
                     raise exc_info[1]
-                except BaseException:  # noqa: B036
+                except BaseException:
                     return func(*args, **kwargs)
             else:
                 return func(*args, **kwargs)

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/tb_cli_modules/cli.py

@@ -76,7 +76,7 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 ]
 
 
-@click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})  # noqa: C901
+@click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})
 @click.option(
     "--debug/--no-debug",
     default=False,
@@ -212,7 +212,7 @@ async def cli(
     if ctx.invoked_subcommand == "auth":
         return
 
-    from tinybird.connectors import create_connector  # noqa
+    from tinybird.connectors import create_connector
 
     if gcp_project_id and gcs_bucket and google_application_credentials and not sf_account:
         bq_config = {
@@ -679,7 +679,7 @@ async def push(
     return
 
 
-@cli.command()  # noqa: C901
+@cli.command()
 @click.option(
     "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
 )
@@ -1251,7 +1251,7 @@ def __unpatch_click_output():
     click.secho = __old_click_echo
 
 
-@cli.command(short_help="Learn how to include info about the CLI in your shell PROMPT")  # noqa: C901
+@cli.command(short_help="Learn how to include info about the CLI in your shell PROMPT")
 @click.pass_context
 @coro
 async def prompt(_ctx: Context) -> None:

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/tb_cli_modules/common.py

@@ -1133,17 +1133,15 @@ async def push_data(
         appended_rows = 0
         parser = None
 
-        if "error" in res and res["error"]:
+        if res.get("error"):
             raise CLIException(FeedbackManager.error_exception(error=res["error"]))
-        if "errors" in res and res["errors"]:
+        if res.get("errors"):
             raise CLIException(FeedbackManager.error_exception(error=res["errors"]))
-        if "blocks" in res and res["blocks"]:
+        if res.get("blocks"):
             for block in res["blocks"]:
                 if "process_return" in block and block["process_return"] is not None:
                     process_return = block["process_return"][0]
-                    parser = (
-                        process_return["parser"] if "parser" in process_return and process_return["parser"] else parser
-                    )
+                    parser = process_return["parser"] if process_return.get("parser") else parser
                     if parser and parser != "clickhouse":
                         parser = process_return["parser"]
                         appended_rows += process_return["lines"]
@@ -1214,12 +1212,12 @@ def autocomplete_topics(ctx: Context, args, incomplete):
 
 
 def validate_datasource_name(name):
-    if not isinstance(name, str) or str == "":
+    if not isinstance(name, str) or name == "":
         raise CLIException(FeedbackManager.error_datasource_name())
 
 
 def validate_connection_id(connection_id):
-    if not isinstance(connection_id, str) or str == "":
+    if not isinstance(connection_id, str) or connection_id == "":
         raise CLIException(FeedbackManager.error_datasource_connection_id())
 
 
@@ -1235,7 +1233,7 @@ def validate_kafka_group(group):
 
 def validate_kafka_auto_offset_reset(auto_offset_reset):
     valid_values = {"latest", "earliest", "none"}
-    if not (auto_offset_reset in valid_values):
+    if auto_offset_reset not in valid_values:
         raise CLIException(FeedbackManager.error_kafka_auto_offset_reset())
 
 
@@ -1588,7 +1586,7 @@ async def try_update_config_with_remote(
 def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
     return (
         click.prompt(
-            f"\nCopy the \"admin your@email\" token from {ui_host}/tokens and paste it here { f'OR press enter to use the token from .tinyb file' if default_token else ''}",
+            f"\nCopy the \"admin your@email\" token from {ui_host}/tokens and paste it here { 'OR press enter to use the token from .tinyb file' if default_token else ''}",
             hide_input=True,
             show_default=False,
             default=default_token,
@@ -2056,3 +2054,28 @@ async def create_aws_iamrole_connection(client: TinyB, service: str, connection_
 """
         )
     click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
+
+
+def get_ca_pem_content(ca_pem: Optional[str], filename: Optional[str] = None) -> Optional[str]:
+    if not ca_pem:
+        return None
+
+    def is_valid_content(text_content: str) -> bool:
+        return text_content.startswith("-----BEGIN CERTIFICATE-----")
+
+    ca_pem_content = ca_pem
+    base_path = Path(getcwd(), filename).parent if filename else Path(getcwd())
+    ca_pem_path = Path(base_path, ca_pem)
+    path_exists = os.path.exists(ca_pem_path)
+
+    if not path_exists:
+        raise CLIConnectionException(FeedbackManager.error_connection_ca_pem_not_found(ca_pem=ca_pem))
+
+    if ca_pem.endswith(".pem") and path_exists:
+        with open(ca_pem_path, "r") as f:
+            ca_pem_content = f.read()
+
+    if not is_valid_content(ca_pem_content):
+        raise CLIConnectionException(FeedbackManager.error_connection_invalid_ca_pem())
+
+    return ca_pem_content

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/tb_cli_modules/connection.py

@@ -21,6 +21,7 @@ from tinybird.tb_cli_modules.common import (
     coro,
     create_aws_iamrole_connection,
     echo_safe_humanfriendly_tables_format_smart_table,
+    get_ca_pem_content,
     validate_aws_iamrole_connection_name,
     validate_aws_iamrole_integration,
     validate_connection_name,
@@ -44,6 +45,7 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
         "kafka_security_protocol",
         "kafka_sasl_mechanism",
         "kafka_schema_registry_url",
+        "kafka_ssl_ca_pem",
     ],
     DataConnectorType.GCLOUD_SCHEDULER: ["gcscheduler_region"],
     DataConnectorType.SNOWFLAKE: [
@@ -131,6 +133,7 @@ def connection_create(ctx: Context) -> None:
     default="PLAIN",
     help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
 )
+@click.option("--ssl-ca-pem", default=None, help="Path or content of the CA Certificate file in PEM format")
 @click.pass_context
 @coro
 async def connection_create_kafka(
@@ -142,6 +145,7 @@ async def connection_create_kafka(
     auto_offset_reset: Optional[str],
     schema_registry_url: Optional[str],
     sasl_mechanism: Optional[str],
+    ssl_ca_pem: Optional[str],
 ) -> None:
     """
     Add a Kafka connection
@@ -174,7 +178,14 @@ async def connection_create_kafka(
     client: TinyB = obj["client"]
 
     result = await client.connection_create_kafka(
-        bootstrap_servers, key, secret, connection_name, auto_offset_reset, schema_registry_url, sasl_mechanism
+        bootstrap_servers,
+        key,
+        secret,
+        connection_name,
+        auto_offset_reset,
+        schema_registry_url,
+        sasl_mechanism,
+        get_ca_pem_content(ssl_ca_pem),
     )
 
     id = result["id"]

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/tb_cli_modules/datasource.py

@@ -229,7 +229,7 @@ async def datasource_replace(
 
     - Replace from local file `tb datasource replace [datasource_name] /path/to/local/file --sql-condition "country='ES'"`
 
-    - Replace from connector `tb datasource replace [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from] --sql-condition "country='ES'"`
+    - Replace from connector `tb datasource replace [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from] --sql-condition "country='ES'"`
     """
 
     if not url and not connector:

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird/tornado_template.py

@@ -848,7 +848,7 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
             # When there are more than 2 curlies in a row, use the
             # innermost ones.  This is useful when generating languages
             # like latex where curlies are also meaningful
-            if curly + 2 < reader.remaining() and reader[curly + 1] == "{" and reader[curly + 2] == "{":  # noqa: W504
+            if curly + 2 < reader.remaining() and reader[curly + 1] == "{" and reader[curly + 2] == "{":
                 curly += 1
                 continue
             break

{tinybird-cli-5.9.1.dev2 → tinybird-cli-5.10.0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev2
+Version: 5.10.0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,11 +18,13 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.9.1.dev2
+5.10.0
 ***********
 
-- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
 - `Added` new "File not found" error to `tb check` when including files from missing paths.
+- `Added` support for Kafka Data Sources with CA certificate.
+- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Fixed` Correctly parse lambda expressions in indexes
 
 5.9.0
 ***********