tinybird-cli 5.9.1.dev3__tar.gz → 5.10.1.dev0__tar.gz

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev3
+Version: 5.10.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,16 +18,13 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.9.1.dev3
+5.10.0
 ***********
 
-- `Fixed` Correctly parse lambda expressions in indexes
-
-5.9.1.dev2
-***********
-
-- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
 - `Added` new "File not found" error to `tb check` when including files from missing paths.
+- `Added` support for Kafka Data Sources with CA certificate.
+- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Fixed` Correctly parse lambda expressions in indexes
 
 5.9.0
 ***********

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.9.1.dev3'
-__revision__ = '6e69bc3'
+__version__ = '5.10.1.dev0'
+__revision__ = 'f8aa90b'

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/client.py

@@ -111,7 +111,7 @@ class TinyB(object):
         retries: int = LIMIT_RETRIES,
         use_token: Optional[str] = None,
         **kwargs,
-    ):  # noqa: C901
+    ):
         url = f"{self.host.strip('/')}/{endpoint.strip('/')}"
 
         token_to_use = use_token if use_token else self.token
@@ -275,7 +275,7 @@ class TinyB(object):
             bigquery_connection = (
                 await self.bigquery_connection() if connector == "bigquery" or connector is None else None
             )
-        connectors = connectors + [bigquery_connection] if bigquery_connection else connectors
+        connectors = [*connectors, bigquery_connection] if bigquery_connection else connectors
         if connector:
             return [
                 {
@@ -865,6 +865,7 @@ class TinyB(object):
         kafka_auto_offset_reset=None,
         kafka_schema_registry_url=None,
         kafka_sasl_mechanism="PLAIN",
+        kafka_ssl_ca_pem=None,
     ):
         params = {
             "service": "kafka",
@@ -880,7 +881,8 @@ class TinyB(object):
             params["kafka_schema_registry_url"] = kafka_schema_registry_url
         if kafka_auto_offset_reset:
             params["kafka_auto_offset_reset"] = kafka_auto_offset_reset
-
+        if kafka_ssl_ca_pem:
+            params["kafka_ssl_ca_pem"] = kafka_ssl_ca_pem
         connection_params = {key: value for key, value in params.items() if value is not None}
 
         return await self._req(

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/datafile.py

@@ -60,7 +60,13 @@ from toposort import toposort
 from tinybird.config import PROJECT_PATHS
 from tinybird.sql_template_fmt import DEFAULT_FMT_LINE_LENGTH, format_sql_template
 from tinybird.syncasync import sync_to_async
-from tinybird.tb_cli_modules.common import _get_tb_client, get_current_main_workspace, getenv_bool, wait_job
+from tinybird.tb_cli_modules.common import (
+    _get_tb_client,
+    get_ca_pem_content,
+    get_current_main_workspace,
+    getenv_bool,
+    wait_job,
+)
 from tinybird.tb_cli_modules.config import CLIConfig
 from tinybird.tb_cli_modules.exceptions import CLIGitReleaseException, CLIPipeException
 
@@ -72,7 +78,7 @@ from .sql_template import get_template_and_variables, get_used_tables_in_templat
 from .tornado_template import UnClosedIfError
 
 os.environ["GIT_PYTHON_REFRESH"] = "quiet"
-from git import HEAD, Diff, GitCommandError, InvalidGitRepositoryError, Repo  # noqa: E402
+from git import HEAD, Diff, GitCommandError, InvalidGitRepositoryError, Repo
 
 INTERNAL_TABLES: Tuple[str, ...] = (
     "datasources_ops_log",
@@ -960,7 +966,7 @@ def parse(
     basepath: str = ".",
     replace_includes: bool = True,
     skip_eval: bool = False,
-) -> Datafile:  # noqa: C901
+) -> Datafile:
     """
     Parses `s` string into a document
     >>> d = parse("FROM SCRATCH\\nSOURCE 'https://example.com'\\n#this is a comment\\nMAINTAINER 'rambo' #this is me\\nNODE \\"test_01\\"\\n    DESCRIPTION this is a node that does whatever\\nSQL >\\n\\n        SELECT * from test_00\\n\\n\\nNODE \\"test_02\\"\\n    DESCRIPTION this is a node that does whatever\\nSQL >\\n\\n    SELECT * from test_01\\n    WHERE a > 1\\n    GROUP by a\\n")
@@ -1110,9 +1116,10 @@ def parse(
                     pass
                 finally:
                     file.seek(0)
-                lines[lineno : lineno + 1] = [""] + list(
-                    StringIO(Template(file.read()).safe_substitute(attrs), newline=None)
-                )
+                lines[lineno : lineno + 1] = [
+                    "",
+                    *list(StringIO(Template(file.read()).safe_substitute(attrs), newline=None)),
+                ]
         except FileNotFoundError:
             raise IncludeFileNotFoundException(f, lineno)
 
@@ -1211,6 +1218,7 @@ def parse(
         "kafka_store_headers": assign_var("kafka_store_headers"),
         "kafka_store_binary_headers": assign_var("kafka_store_binary_headers"),
         "kafka_key_avro_deserialization": assign_var("kafka_key_avro_deserialization"),
+        "kafka_ssl_ca_pem": assign_var("kafka_ssl_ca_pem"),
         "import_service": assign_var("import_service"),
         "import_connection_name": assign_var("import_connection_name"),
         "import_schedule": assign_var("import_schedule"),
@@ -1318,7 +1326,7 @@ async def process_file(
     workspace_map: Optional[Dict] = None,
     workspace_lib_paths: Optional[List[Tuple[str, str]]] = None,
     current_ws: Optional[Dict[str, Any]] = None,
-):  # noqa: C901 B006
+):
     if workspace_map is None:
         workspace_map = {}
 
@@ -1349,6 +1357,7 @@ async def process_file(
                     "kafka_connection_name": params.get("kafka_connection_name", None),
                     "kafka_auto_offset_reset": params.get("kafka_auto_offset_reset", None),
                     "kafka_schema_registry_url": params.get("kafka_schema_registry_url", None),
+                    "kafka_ssl_ca_pem": get_ca_pem_content(params.get("kafka_ssl_ca_pem", None), filename),
                 }
 
                 connector = await tb_client.get_connection(**connector_params)
@@ -2868,7 +2877,7 @@ async def new_pipe(
     config: Any = None,
     fork_downstream: Optional[bool] = False,
     fork: Optional[bool] = False,
-):  # noqa: C901
+):
     # TODO use tb_client instead of calling the urls directly.
     host = tb_client.host
     token = tb_client.token
@@ -3270,10 +3279,10 @@ async def new_ds(
                     if job_url:
                         click.echo(FeedbackManager.success_dynamodb_initial_load(job_url=job_url))
 
-            if "tokens" in ds and ds["tokens"]:
+            if ds.get("tokens"):
                 await manage_tokens()
 
-            if "shared_with" in ds and ds["shared_with"]:
+            if ds.get("shared_with"):
                 if not user_token:
                     click.echo(FeedbackManager.info_skipping_shared_with_entry())
                 else:
@@ -4189,7 +4198,7 @@ async def folder_push(
     use_main: bool = False,
     check_outdated: bool = True,
     hide_folders: bool = False,
-):  # noqa: C901
+):
     workspaces: List[Dict[str, Any]] = (await tb_client.user_workspaces_and_branches()).get("workspaces", [])
     current_ws: Dict[str, Any] = next(
         (workspace for workspace in workspaces if config and workspace.get("id", ".") == config.get("id", "..")), {}
@@ -4754,7 +4763,7 @@ DATAFILE_INDENT = " " * 4
 
 
 def format_schema(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
-    if "schema" in node and node["schema"]:
+    if node.get("schema"):
         file_parts.append("SCHEMA >")
         file_parts.append(DATAFILE_NEW_LINE)
         columns = schema_to_sql_columns(node["columns"])
@@ -4766,7 +4775,7 @@ def format_schema(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
 
 
 def format_indices(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
-    if "indexes" in node and node["indexes"]:
+    if node.get("indexes"):
         indexes = node["indexes"]
         file_parts.append("INDEXES >")
         file_parts.append(DATAFILE_NEW_LINE)
@@ -4995,7 +5004,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
             file_parts.append(DATAFILE_NEW_LINE)
             file_parts.append(f'COPY_MODE {node.get("mode")}')
 
-        if CopyParameters.COPY_SCHEDULE in node and node[CopyParameters.COPY_SCHEDULE]:
+        if node.get(CopyParameters.COPY_SCHEDULE):
             is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
             file_parts.append(DATAFILE_NEW_LINE)
             file_parts.append(
@@ -5154,7 +5163,7 @@ async def folder_pull(
     verbose: bool = True,
     progress_bar: bool = False,
     fmt: bool = False,
-):  # noqa: C901
+):
     pattern = re.compile(match) if match else None
 
     def _get_latest_versions(resources: List[str]):
@@ -5321,7 +5330,7 @@ async def diff_command(
 
     if filenames:
         if len(filenames) == 1:
-            filenames = [filenames[0]] + get_project_filenames(filenames[0])
+            filenames = [filenames[0], *get_project_filenames(filenames[0])]
         await folder_pull(client, target_dir, False, None, True, verbose=False)
     else:
         filenames = get_project_filenames(".")

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/feedback_manager.py

@@ -238,6 +238,8 @@ class FeedbackManager:
     error_connection_does_not_exists = error_message("Connection {connection_id} does not exist")
     error_connection_create = error_message("Connection {connection_name} could not be created: {error}")
     error_connection_integration_not_available = error_message("Connection could not be created: {error}")
+    error_connection_invalid_ca_pem = error_message("Invalid CA certificate in PEM format")
+    error_connection_ca_pem_not_found = error_message("CA certificate in PEM format not found at {ca_pem}")
     error_workspace = error_message("Workspace {workspace} not found. use 'tb workspace ls' to list your workspaces")
     error_deleted_include = error_message(
         "Related include file {include_file} was deleted and it's used in {filename}. Delete or remove dependency from {filename}."

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/sql.py

@@ -526,7 +526,7 @@ REGEX_WHITESPACE = re.compile(r"\s*")
 REGEX_COMMENT = re.compile(r"\-\-[^\n\r]*[\n\r]")
 
 
-def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:  # noqa: C901
+def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
     # CH syntax from https://clickhouse.com/docs/en/sql-reference/statements/create/table/
     # name1 [type1] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|ALIAS expr1] [compression_codec] [TTL expr1]
     try:

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/sql_template.py

@@ -359,7 +359,7 @@ def defined(x=None):
     return True
 
 
-def array_type(types):  # noqa: C901
+def array_type(types):
     def _f(
         x, _type=None, default=None, defined=True, required=None, description=None, enum=None, example=None, format=None
     ):
@@ -1370,7 +1370,7 @@ _namespace = {
 }
 
 
-reserved_vars = set(["_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error"] + list(vars(builtins)))
+reserved_vars = set(["_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))])
 for p in DEFAULT_PARAM_NAMES:  # we handle these in an specific manner
     reserved_vars.discard(p)  # `format` is part of builtins
 error_vars = ["error", "custom_error"]
@@ -1552,7 +1552,7 @@ def get_var_names(t):
         raise SQLTemplateException(e)
 
 
-def get_var_data(content, node_id=None):  # noqa: C901
+def get_var_data(content, node_id=None):
     def node_to_value(x):
         if type(x) in (ast.Bytes, ast.Str):
             return x.s
@@ -1703,7 +1703,7 @@ def get_var_data(content, node_id=None):  # noqa: C901
     return [dict(name=k, **v) for k, v in vars.items()]
 
 
-def get_var_names_and_types(t, node_id=None):  # noqa: C901
+def get_var_names_and_types(t, node_id=None):
     """
     >>> get_var_names_and_types(Template("SELECT * FROM filter_value WHERE description = {{Float32(with_value, 0.0)}}"))
     [{'name': 'with_value', 'type': 'Float32', 'default': 0.0}]

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/syncasync.py

@@ -177,7 +177,7 @@ class _WorkItem:
             return
         try:
             result = self.fn(*self.args, **self.kwargs)
-        except BaseException as exc:  # noqa: B036
+        except BaseException as exc:
             self.future.set_exception(exc)
             # Break a reference cycle with the exception 'exc'
             self = None
@@ -473,11 +473,11 @@ class AsyncToSync:
             if exc_info[1]:
                 try:
                     raise exc_info[1]
-                except BaseException:  # noqa: B036
+                except BaseException:
                     result = await self.awaitable(*args, **kwargs)
             else:
                 result = await self.awaitable(*args, **kwargs)
-        except BaseException as e:  # noqa: B036
+        except BaseException as e:
             call_result.set_exception(e)
         else:
             call_result.set_result(result)
@@ -639,7 +639,7 @@ class SyncToAsync:
             if exc_info[1]:
                 try:
                     raise exc_info[1]
-                except BaseException:  # noqa: B036
+                except BaseException:
                     return func(*args, **kwargs)
             else:
                 return func(*args, **kwargs)

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/tb_cli_modules/cli.py

@@ -76,7 +76,7 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 ]
 
 
-@click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})  # noqa: C901
+@click.group(cls=CatchAuthExceptions, context_settings={"help_option_names": ["-h", "--help"]})
 @click.option(
     "--debug/--no-debug",
     default=False,
@@ -212,7 +212,7 @@ async def cli(
     if ctx.invoked_subcommand == "auth":
         return
 
-    from tinybird.connectors import create_connector  # noqa
+    from tinybird.connectors import create_connector
 
     if gcp_project_id and gcs_bucket and google_application_credentials and not sf_account:
         bq_config = {
@@ -679,7 +679,7 @@ async def push(
     return
 
 
-@cli.command()  # noqa: C901
+@cli.command()
 @click.option(
     "--folder", default=None, type=click.Path(exists=True, file_okay=False), help="Folder where files will be placed"
 )
@@ -1251,7 +1251,7 @@ def __unpatch_click_output():
     click.secho = __old_click_echo
 
 
-@cli.command(short_help="Learn how to include info about the CLI in your shell PROMPT")  # noqa: C901
+@cli.command(short_help="Learn how to include info about the CLI in your shell PROMPT")
 @click.pass_context
 @coro
 async def prompt(_ctx: Context) -> None:

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/tb_cli_modules/common.py

@@ -1133,17 +1133,15 @@ async def push_data(
         appended_rows = 0
         parser = None
 
-        if "error" in res and res["error"]:
+        if res.get("error"):
             raise CLIException(FeedbackManager.error_exception(error=res["error"]))
-        if "errors" in res and res["errors"]:
+        if res.get("errors"):
             raise CLIException(FeedbackManager.error_exception(error=res["errors"]))
-        if "blocks" in res and res["blocks"]:
+        if res.get("blocks"):
             for block in res["blocks"]:
                 if "process_return" in block and block["process_return"] is not None:
                     process_return = block["process_return"][0]
-                    parser = (
-                        process_return["parser"] if "parser" in process_return and process_return["parser"] else parser
-                    )
+                    parser = process_return["parser"] if process_return.get("parser") else parser
                     if parser and parser != "clickhouse":
                         parser = process_return["parser"]
                         appended_rows += process_return["lines"]
@@ -2056,3 +2054,28 @@ async def create_aws_iamrole_connection(client: TinyB, service: str, connection_
 """
         )
     click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
+
+
+def get_ca_pem_content(ca_pem: Optional[str], filename: Optional[str] = None) -> Optional[str]:
+    if not ca_pem:
+        return None
+
+    def is_valid_content(text_content: str) -> bool:
+        return text_content.startswith("-----BEGIN CERTIFICATE-----")
+
+    ca_pem_content = ca_pem
+    base_path = Path(getcwd(), filename).parent if filename else Path(getcwd())
+    ca_pem_path = Path(base_path, ca_pem)
+    path_exists = os.path.exists(ca_pem_path)
+
+    if not path_exists:
+        raise CLIConnectionException(FeedbackManager.error_connection_ca_pem_not_found(ca_pem=ca_pem))
+
+    if ca_pem.endswith(".pem") and path_exists:
+        with open(ca_pem_path, "r") as f:
+            ca_pem_content = f.read()
+
+    if not is_valid_content(ca_pem_content):
+        raise CLIConnectionException(FeedbackManager.error_connection_invalid_ca_pem())
+
+    return ca_pem_content

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/tb_cli_modules/connection.py

@@ -21,6 +21,7 @@ from tinybird.tb_cli_modules.common import (
     coro,
     create_aws_iamrole_connection,
     echo_safe_humanfriendly_tables_format_smart_table,
+    get_ca_pem_content,
     validate_aws_iamrole_connection_name,
     validate_aws_iamrole_integration,
     validate_connection_name,
@@ -44,6 +45,7 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
         "kafka_security_protocol",
         "kafka_sasl_mechanism",
         "kafka_schema_registry_url",
+        "kafka_ssl_ca_pem",
     ],
     DataConnectorType.GCLOUD_SCHEDULER: ["gcscheduler_region"],
     DataConnectorType.SNOWFLAKE: [
@@ -131,6 +133,7 @@ def connection_create(ctx: Context) -> None:
     default="PLAIN",
     help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
 )
+@click.option("--ssl-ca-pem", default=None, help="Path or content of the CA Certificate file in PEM format")
 @click.pass_context
 @coro
 async def connection_create_kafka(
@@ -142,6 +145,7 @@ async def connection_create_kafka(
     auto_offset_reset: Optional[str],
     schema_registry_url: Optional[str],
     sasl_mechanism: Optional[str],
+    ssl_ca_pem: Optional[str],
 ) -> None:
     """
     Add a Kafka connection
@@ -174,7 +178,14 @@ async def connection_create_kafka(
     client: TinyB = obj["client"]
 
     result = await client.connection_create_kafka(
-        bootstrap_servers, key, secret, connection_name, auto_offset_reset, schema_registry_url, sasl_mechanism
+        bootstrap_servers,
+        key,
+        secret,
+        connection_name,
+        auto_offset_reset,
+        schema_registry_url,
+        sasl_mechanism,
+        get_ca_pem_content(ssl_ca_pem),
     )
 
     id = result["id"]

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/tb_cli_modules/datasource.py

@@ -229,7 +229,7 @@ async def datasource_replace(
 
     - Replace from local file `tb datasource replace [datasource_name] /path/to/local/file --sql-condition "country='ES'"`
 
-    - Replace from connector `tb datasource replace [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from] --sql-condition "country='ES'"`
+    - Replace from connector `tb datasource replace [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from] --sql-condition "country='ES'"`
     """
 
     if not url and not connector:

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird/tornado_template.py

@@ -848,7 +848,7 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
             # When there are more than 2 curlies in a row, use the
             # innermost ones.  This is useful when generating languages
             # like latex where curlies are also meaningful
-            if curly + 2 < reader.remaining() and reader[curly + 1] == "{" and reader[curly + 2] == "{":  # noqa: W504
+            if curly + 2 < reader.remaining() and reader[curly + 1] == "{" and reader[curly + 2] == "{":
                 curly += 1
                 continue
             break

{tinybird-cli-5.9.1.dev3 → tinybird-cli-5.10.1.dev0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.9.1.dev3
+Version: 5.10.1.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,16 +18,13 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-5.9.1.dev3
+5.10.0
 ***********
 
-- `Fixed` Correctly parse lambda expressions in indexes
-
-5.9.1.dev2
-***********
-
-- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
 - `Added` new "File not found" error to `tb check` when including files from missing paths.
+- `Added` support for Kafka Data Sources with CA certificate.
+- `Changed` Upgrade clickhouse-toolset to 0.32.dev0
+- `Fixed` Correctly parse lambda expressions in indexes
 
 5.9.0
 ***********