tinybird-cli 5.3.1.dev3__tar.gz → 5.3.1.dev5__tar.gz

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.3.1.dev3
+Version: 5.3.1.dev5
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
 Author-email: support@tinybird.co
-Requires-Python: >=3.8, <3.12
+Requires-Python: >=3.8, <3.13
 Description-Content-Type: text/x-rst
 Provides-Extra: bigquery
 Provides-Extra: snowflake
@@ -18,6 +18,21 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.3.1.dev5
+**********
+
+- `Fixed` Do not try to deploy shared Datafiles from the `vendor` folder on `tb deploy`
+
+5.3.1.dev4
+**********
+
+- `Added` support to Python 3.12
+
+5.3.1.dev3
+**********
+
+- `Added` support to Stream Pipes in `tb push` and `tb pull` commands. (Feature in development. It requires access)
+
 5.3.1.dev2
 **********
 

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.3.1.dev3'
-__revision__ = '00ee83f'
+__version__ = '5.3.1.dev5'
+__revision__ = '4613f9c'

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/client.py

@@ -600,6 +600,9 @@ class TinyB(object):
     async def pipe_remove_sink(self, pipe_name_or_id: str, node_id: str):
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/sink", method="DELETE")
 
+    async def pipe_remove_stream(self, pipe_name_or_id: str, node_id: str):
+        return await self._req(f"/v0/pipes/{pipe_name_or_id}/nodes/{node_id}/stream", method="DELETE")
+
     async def pipe_run_sink(self, pipe_name_or_id: str, params: Optional[Dict[str, str]] = None):
         params = {**params} if params else {}
         return await self._req(f"/v0/pipes/{pipe_name_or_id}/sink?{urlencode(params)}", method="POST")

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/datafile.py

@@ -95,6 +95,7 @@ class PipeTypes:
     ENDPOINT = "endpoint"
     COPY = "copy"
     DATA_SINK = "sink"
+    STREAM = "stream"
     DEFAULT = "default"
 
 
@@ -105,6 +106,7 @@ class PipeNodeTypes:
     DEFAULT = "default"
     DATA_SINK = "sink"
     COPY = "copy"
+    STREAM = "stream"
 
 
 class DataFileExtensions:
@@ -190,8 +192,8 @@ class ImportReplacements:
 
 
 class ExportReplacements:
-    SERVICES = ("gcs_hmac", "s3", "s3_iamrole")
-    NODE_TYPES = PipeNodeTypes.DATA_SINK
+    SERVICES = ("gcs_hmac", "s3", "s3_iamrole", "kafka")
+    NODE_TYPES = (PipeNodeTypes.DATA_SINK, PipeNodeTypes.STREAM)
     _REPLACEMENTS = (
         ("export_service", "service", None),
         ("export_connection_name", "connection", None),
@@ -201,11 +203,20 @@ class ExportReplacements:
         ("export_format", "format", "csv"),
         ("export_compression", "compression", None),
         ("export_strategy", "strategy", "@new"),
+        ("kafka_connection_name", "connection", None),
+        ("kafka_topic", "kafka_topic", None),
+        ("kafka_group_id", "kafka_group_id", None),
     )
 
+    @staticmethod
+    def get_export_service(node: Dict[str, Optional[str]]) -> str:
+        if (node.get("type", "standard") or "standard").lower() == PipeNodeTypes.STREAM:
+            return "kafka"
+        return (node.get("export_service", "") or "").lower()
+
     @staticmethod
     def is_export_node(node: Dict[str, Optional[str]]) -> bool:
-        export_service = node.get("export_service", None)
+        export_service = ExportReplacements.get_export_service(node)
         node_type = (node.get("type", "standard") or "standard").lower()
         if not export_service:
             return False
@@ -219,16 +230,26 @@ class ExportReplacements:
     def get_params_from_datafile(node: Dict[str, Optional[str]]) -> Dict[str, Optional[str]]:
         """Returns the export parameters for a given node."""
         params = {}
+        export_service = ExportReplacements.get_export_service(node)
         for datafile_key, export_key, default_value in ExportReplacements._REPLACEMENTS:
+            if export_service != "kafka" and "kafka_" in datafile_key:
+                continue
+            if export_service == "kafka" and "export_" in datafile_key:
+                continue
             if datafile_key == "export_schedule" and node.get(datafile_key, None) == ON_DEMAND:
                 node[datafile_key] = ""
             params[export_key] = node[datafile_key] if datafile_key in node else default_value
         return params
 
     @staticmethod
-    def get_datafile_key(param: str) -> Optional[str]:
+    def get_datafile_key(param: str, node: Dict[str, Optional[str]]) -> Optional[str]:
         """Returns the datafile key for a given export parameter."""
+        export_service = ExportReplacements.get_export_service(node)
         for datafile_k, export_k, _ in ExportReplacements._REPLACEMENTS:
+            if export_service != "kafka" and "kafka_" in datafile_k:
+                continue
+            if export_service == "kafka" and "export_" in datafile_k:
+                continue
             if export_k == param.lower():
                 return datafile_k.upper()
         return None
@@ -2323,6 +2344,8 @@ async def check_pipe(
             await cl.pipe_remove_copy(current_pipe["id"], current_pipe["copy_node"])
         if pipe_type == PipeTypes.DATA_SINK:
             await cl.pipe_remove_sink(current_pipe["id"], current_pipe["sink_node"])
+        if pipe_type == PipeTypes.STREAM:
+            await cl.pipe_remove_stream(current_pipe["id"], current_pipe["stream_node"])
 
     # In case of doing --force for a materialized view, checker is being created as standard pipe
     for node in checker_pipe["nodes"]:
@@ -2525,6 +2548,9 @@ async def check_copy_pipe(pipe, copy_node, tb_client: TinyB):
     if pipe_type == PipeTypes.DATA_SINK:
         await tb_client.pipe_remove_sink(pipe_name, pipe["sink_node"])
 
+    if pipe_type == PipeTypes.STREAM:
+        await tb_client.pipe_remove_stream(pipe_name, pipe["stream_node"])
+
 
 async def check_sink_pipe(pipe, sink_node, tb_client: TinyB):
     if not sink_node["export_params"]:
@@ -2548,6 +2574,29 @@ async def check_sink_pipe(pipe, sink_node, tb_client: TinyB):
     if pipe_type == PipeTypes.COPY:
         await tb_client.pipe_remove_copy(pipe_name, pipe["copy_node"])
 
+    if pipe_type == PipeTypes.STREAM:
+        await tb_client.pipe_remove_stream(pipe_name, pipe["stream_node"])
+
+
+async def check_stream_pipe(pipe, stream_node, tb_client: TinyB):
+    if not stream_node["params"]:
+        return
+
+    if not pipe:
+        return
+
+    pipe_name = pipe["name"]
+    pipe_type = pipe["type"]
+
+    if pipe_type == PipeTypes.ENDPOINT:
+        await tb_client.pipe_remove_endpoint(pipe_name, pipe["endpoint"])
+
+    if pipe_type == PipeTypes.COPY:
+        await tb_client.pipe_remove_copy(pipe_name, pipe["copy_node"])
+
+    if pipe_type == PipeTypes.DATA_SINK:
+        await tb_client.pipe_remove_sink(pipe_name, pipe["sink_node"])
+
 
 def show_materialized_view_warnings(warnings):
     """
@@ -2635,6 +2684,7 @@ async def new_pipe(
     is_materialized = any([node.get("params", {}).get("type", None) == "materialized" for node in p["nodes"]])
     copy_node = next((node for node in p["nodes"] if node.get("params", {}).get("type", None) == "copy"), None)
     sink_node = next((node for node in p["nodes"] if node.get("params", {}).get("type", None) == "sink"), None)
+    stream_node = next((node for node in p["nodes"] if node.get("params", {}).get("type", None) == "stream"), None)
 
     for node in p["nodes"]:
         if node["params"]["name"] == p["name"]:
@@ -2644,7 +2694,7 @@ async def new_pipe(
         if force or run_tests:
             # TODO: this should create a different node and rename it to the final one on success
             if check and not populate:
-                if not is_materialized and not copy_node and not sink_node:
+                if not is_materialized and not copy_node and not sink_node and not stream_node:
                     await check_pipe(
                         p,
                         host,
@@ -2680,6 +2730,8 @@ async def new_pipe(
                         await check_copy_pipe(pipe=current_pipe, copy_node=copy_node, tb_client=tb_client)
                     if sink_node:
                         await check_sink_pipe(pipe=current_pipe, sink_node=sink_node, tb_client=tb_client)
+                    if stream_node:
+                        await check_stream_pipe(pipe=current_pipe, stream_node=stream_node, tb_client=tb_client)
             if run_tests:
                 logging.info(f"skipping force override of {p['name']}")
                 return
@@ -2727,6 +2779,9 @@ async def new_pipe(
     if sink_node:
         body.update(sink_node.get("export_params", {}))
 
+    if stream_node:
+        body.update(stream_node.get("export_params", {}))
+
     post_headers = {"Content-Type": "application/json"}
 
     post_headers.update(headers)
@@ -3804,6 +3859,10 @@ async def build_graph(
                                 processed_filename = f.relative_to(os.getcwd())
                             except ValueError:
                                 processed_filename = f
+                            # This is to avoid processing shared data sources
+                            if "vendor/" in str(processed_filename):
+                                click.echo(FeedbackManager.info_skipping_resource(resource=processed_filename))
+                                continue
                             click.echo(FeedbackManager.info_processing_file(filename=processed_filename))
                         await process(str(f), deps, dep_map, to_run, workspace_lib_paths)
 
@@ -4703,7 +4762,6 @@ async def format_engine(
 async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[str]:
     node_type = node.get("type", "").lower()
     node_type_upper = f"TYPE {node_type.upper()}"
-
     # Materialized pipe
     if node_type == PipeNodeTypes.MATERIALIZED:
         file_parts.append(node_type_upper)
@@ -4732,7 +4790,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
             file_parts.append(f"{CopyParameters.COPY_SCHEDULE.upper()} {ON_DEMAND}")
         file_parts.append(DATAFILE_NEW_LINE)
 
-    # Sink pipe
+    # Sink or Stream pipe
     if ExportReplacements.is_export_node(node):
         file_parts.append(node_type_upper)
         export_params = ExportReplacements.get_params_from_datafile(node)
@@ -4740,7 +4798,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
         for param, value in export_params.items():
             if param == "schedule_cron" and not value:
                 value = ON_DEMAND
-            datafile_key = ExportReplacements.get_datafile_key(param)
+            datafile_key = ExportReplacements.get_datafile_key(param, node)
             if datafile_key:
                 file_parts.append(f"{datafile_key} {value}")
                 file_parts.append(DATAFILE_NEW_LINE)
@@ -5189,6 +5247,7 @@ def is_endpoint_with_no_dependencies(
             PipeNodeTypes.MATERIALIZED,
             PipeNodeTypes.COPY,
             PipeNodeTypes.DATA_SINK,
+            PipeNodeTypes.STREAM,
         ]:
             return False
 

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/feedback_manager.py

@@ -709,6 +709,7 @@ Ready? """
     info_unlinking_materialized_pipe = info_message("** Unlinking materialized pipe {pipe}")
     info_unlinking_copy_pipe = info_message("** Unlinking copy pipe {pipe}")
     info_unlinking_sink_pipe = info_message("** Unlinking sink pipe {pipe}")
+    info_unlinking_stream_pipe = info_message("** Unlinking stream pipe {pipe}")
     info_materialized_unlinking_pipe_not_found = info_message("** {pipe} not found")
     info_materialized_dry_unlinking_pipe = info_message("** [DRY RUN] Unlinking materialized pipe {pipe}")
     info_copy_datasource_created = info_message("** Copy pipe '{pipe}' created the Data Source '{datasource}'")

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/sql_template.py

@@ -1429,6 +1429,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
             "__loader__": ObjectDict(get_source=lambda name: self.code),
             "max_threads": set_max_threads,
             "tb_secret": set_tb_secret,
+            "tb_var": set_tb_secret,
             "backend_hint": set_backend_hint,
             "cache_ttl": set_cache_ttl,
             "activate": set_activate,
@@ -2116,6 +2117,8 @@ def render_sql_template(
     ("select * from table where str = 'test'", {}, ['pipeline'])
     >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", secrets = [ 'tb_secret_test' ])
     ('select * from table where str = {test: String}', {}, [])
+    >>> render_sql_template("select * from table where str = {{tb_var('test')}}", secrets = [ 'tb_secret_test' ])
+    ('select * from table where str = {test: String}', {}, [])
     >>> render_sql_template("select * from table where str = {{tb_secret('test')}}", variables = { 'test': '1234' })
     Traceback (most recent call last):
     ...
@@ -2146,7 +2149,7 @@ def render_sql_template(
             return Comment("error launched")
 
         v: dict = {x["name"]: Placeholder(x["name"], x["line"]) for x in template_variables}
-        is_tb_secret = any([s for s in template_variables if s["name"] == "tb_secret"])
+        is_tb_secret = any([s for s in template_variables if s["name"] == "tb_secret" or s["name"] == "tb_var"])
 
         if variables:
             v.update(variables)

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/sql_toolset.py

@@ -70,7 +70,9 @@ def sql_get_used_tables_cached(
     table_functions: bool = True,
     function_allow_list: Optional[FrozenSet[str]] = None,
 ) -> List[Tuple[str, str, str]]:
-    """
+    """More like: get used sql names
+
+    Returns a list of tuples: (database_or_namespace, table_name, table_func).
     >>> sql_get_used_tables("SELECT 1 FROM the_table")
     [('', 'the_table', '')]
     >>> sql_get_used_tables("SELECT 1 FROM the_database.the_table")
@@ -104,6 +106,8 @@ def sql_get_used_tables_cached(
                 raise InvalidFunction(msg=msg) from e
             elif "Unknown function tb_secret" in msg:
                 raise InvalidFunction(msg="Unknown function tb_secret. Usage: {{tb_secret('secret_name')}}") from e
+            elif "Unknown function tb_var" in msg:
+                raise InvalidFunction(msg="Unknown function tb_var. Usage: {{tb_var('var_name')}}") from e
             raise
         return [(default_database, sql, "")]
 
@@ -115,6 +119,10 @@ def sql_get_used_tables(
     table_functions: bool = True,
     function_allow_list: Optional[FrozenSet[str]] = None,
 ) -> List[Tuple[str, str, str]]:
+    """More like: get used sql names
+
+    Returns a list of tuples: (database_or_namespace, table_name, table_func).
+    """
     hashable_list = frozenset() if function_allow_list is None else function_allow_list
 
     return copy.copy(

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/tb_cli_modules/pipe.py

@@ -16,7 +16,7 @@ from click import Context
 import tinybird.context as context
 from tinybird.client import DoesNotExistException, TinyB
 from tinybird.config import DEFAULT_API_HOST, FeatureFlags
-from tinybird.datafile import PipeTypes, folder_push, get_name_version, process_file, wait_job
+from tinybird.datafile import PipeNodeTypes, PipeTypes, folder_push, get_name_version, process_file, wait_job
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.branch import warn_if_in_live
 from tinybird.tb_cli_modules.cli import cli
@@ -348,6 +348,16 @@ async def pipe_unlink_output_node(
                 await client.pipe_remove_sink(pipe["name"], node_uid)
                 click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
 
+        if pipe["type"] == PipeTypes.STREAM:
+            click.echo(FeedbackManager.info_unlinking_stream_pipe(pipe=pipe["name"]))
+            node_uid = next((node["id"] for node in pipe["nodes"] if node["node_type"] == PipeNodeTypes.STREAM), None)
+
+            if not node_uid:
+                raise CLIPipeException(FeedbackManager.error_unlinking_pipe_not_linked(pipe=pipe_name_or_id))
+            else:
+                await client.pipe_remove_stream(pipe["name"], node_uid)
+                click.echo(FeedbackManager.success_pipe_unlinked(pipe=pipe["name"]))
+
     except Exception as e:
         raise CLIPipeException(FeedbackManager.error_exception(error=e))
 

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird/tornado_template.py

@@ -204,19 +204,15 @@ import os.path
 import posixpath
 import re
 import threading
+from io import StringIO
 from typing import Union
 
 from tornado import escape
 from tornado.log import app_log
-from tornado.util import PY3, ObjectDict, exec_in, unicode_type
+from tornado.util import ObjectDict, exec_in, unicode_type
 
 from .context import disable_template_security_validation
 
-if PY3:
-    from io import StringIO
-else:
-    from cStringIO import StringIO
-
 _DEFAULT_AUTOESCAPE = "xhtml_escape"
 _UNSET = object()
 
@@ -1051,6 +1047,7 @@ VALID_FUNCTION_NAMES = {
     "sql_unescape",
     "max_threads",
     "tb_secret",
+    "tb_var",
     "Int",
     "table",
     "TABLE",

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird_cli.egg-info/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.3.1.dev3
+Version: 5.3.1.dev5
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
 Author-email: support@tinybird.co
-Requires-Python: >=3.8, <3.12
+Requires-Python: >=3.8, <3.13
 Description-Content-Type: text/x-rst
 Provides-Extra: bigquery
 Provides-Extra: snowflake
@@ -18,6 +18,21 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.3.1.dev5
+**********
+
+- `Fixed` Do not try to deploy shared Datafiles from the `vendor` folder on `tb deploy`
+
+5.3.1.dev4
+**********
+
+- `Added` support to Python 3.12
+
+5.3.1.dev3
+**********
+
+- `Added` support to Stream Pipes in `tb push` and `tb pull` commands. (Feature in development. It requires access)
+
 5.3.1.dev2
 **********
 

{tinybird-cli-5.3.1.dev3 → tinybird-cli-5.3.1.dev5}/tinybird_cli.egg-info/requires.txt

@@ -12,7 +12,7 @@ requests<3,>=2.28.1
 shandy-sqlfmt==0.11.1
 shandy-sqlfmt[jinjafmt]==0.11.1
 toposort==1.10
-tornado==5.1.1
+tornado==6.0.0
 urllib3<2,>=1.26.14
 wheel
 packaging<24,>=23.1