tinybird-cli 4.1.1.dev1__tar.gz → 5.0.0.dev0__tar.gz

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 4.1.1.dev1
+Version: 5.0.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,7 +18,18 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-4.1.1.dev0
+5.0.0.dev0
+************
+
+- `Changed` Make `insertion_date`` column explicit. This column is no longer inferred, it must be present in the Data Source file.
+
+4.1.2.dev0
+************
+
+- `Added` parameter to `tb pipe regression-test` --relative-change
+
+
+4.1.1
 ************
 
 - `Fixed` workspace info gathering when switching branches. We were prioritizing the general workspace properties over the user ones.
@@ -27,7 +38,7 @@ Changelog
 ************
 
 - `Added` `tb token create` command to be able to create static and JWT tokens from the CLI. You can check more information at https://www.tinybird.co/blog-posts/jwt-api-endpoints-public-beta
-- `Fixed` `tb init --git` to pin `tinybird-cli>=4,<5` in `requirements.txt` to avoid issues with the latest version of the CLI. 
+- `Fixed` `tb init --git` to pin `tinybird-cli>=4,<5` in `requirements.txt` to avoid issues with the latest version of the CLI.
 
 4.0.0
 ************
@@ -85,7 +96,7 @@ This is a major release, please read the commands affected below and consider up
 ************
 
 - `Changed` fixed major version of tinybird-cli to lower than 4 when using `tb init --git`
-- `Changed` behavior when running `tb deploy` on a branch to push the connection settings to the backend. This change is the backend that decides what to do. 
+- `Changed` behavior when running `tb deploy` on a branch to push the connection settings to the backend. This change is the backend that decides what to do.
 - `Changed` `tb pipe sink` commands are now available
 - `Fixed` regression tests query when filtering by specific parameter
 - `Fixed` Avoid `tb fmt` to error if there's a `CASE` in the sql

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.1.1.dev1'
-__revision__ = '29bdc3e'
+__version__ = '5.0.0.dev0'
+__revision__ = '2147517'

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird/context.py

@@ -7,6 +7,7 @@ hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")
 use_gatherer: ContextVar[bool] = ContextVar("use_gatherer")
 allow_gatherer_fallback: ContextVar[bool] = ContextVar("allow_gatherer_fallback")
+gatherer_allow_s3_backup_on_user_errors: ContextVar[bool] = ContextVar("gatherer_allow_s3_backup_on_user_errors")
 disable_template_security_validation: ContextVar[bool] = ContextVar("disable_template_security_validation")
 origin: ContextVar[str] = ContextVar("origin")
 request_id: ContextVar[str] = ContextVar("request_id")

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird/datafile.py

@@ -773,7 +773,7 @@ def parse_pipe(
         for node in doc.nodes:
             sql = node.get("sql", "")
             if sql.strip()[0] == "%":
-                sql, _ = render_sql_template(sql[1:], test_mode=True, name=node["name"])
+                sql, _, _ = render_sql_template(sql[1:], test_mode=True, name=node["name"])
             # it'll fail with a ModuleNotFoundError when the toolset is not available but it returns the parsed doc
             from tinybird.sql_toolset import format_sql as toolset_format_sql
 
@@ -785,7 +785,7 @@ def parse_pipe(
             )
         )
     except ValueError as e:
-        t, template_variables = get_template_and_variables(sql, name=node["name"])
+        t, template_variables, _ = get_template_and_variables(sql, name=node["name"])
 
         if sql.strip()[0] != "%" and len(template_variables) > 0:
             raise click.ClickException(FeedbackManager.error_template_start(filename=filename))
@@ -1492,7 +1492,7 @@ async def process_file(
             is_template = False
             if sql[0] == "%":
                 try:
-                    sql_rendered, _ = render_sql_template(sql[1:], test_mode=True)
+                    sql_rendered, _, _ = render_sql_template(sql[1:], test_mode=True)
                 except Exception as e:
                     raise click.ClickException(
                         FeedbackManager.error_parsing_node(node=node["name"], pipe=name, error=str(e))
@@ -1766,6 +1766,7 @@ class PipeChecker(unittest.TestCase):
         only_response_times: bool,
         ignore_order: bool,
         validate_processed_bytes: bool,
+        relative_change: float,
         *args,
         **kwargs,
     ) -> None:
@@ -1784,6 +1785,7 @@ class PipeChecker(unittest.TestCase):
         self.only_response_times = only_response_times
         self.ignore_order = ignore_order
         self.validate_processed_bytes = validate_processed_bytes
+        self.relative_change = relative_change
 
         parsed = urlparse(self.current_pipe_url)
         self.checker_pipe_url = f"{parsed.scheme}://{parsed.netloc}/v0/pipes/{self.checker_pipe_name}.json"
@@ -1901,12 +1903,13 @@ class PipeChecker(unittest.TestCase):
         for _, (current_data_e, check_fixtures_data_e) in enumerate(zip(current_data, checker_data)):
             self.assertEqual(list(current_data_e.keys()), list(check_fixtures_data_e.keys()))
             for x in current_data_e.keys():
-                if isinstance(current_data_e[x], float):
+                if isinstance(current_data_e[x], (float, int)):
                     d = abs(current_data_e[x] - check_fixtures_data_e[x])
+
                     try:
                         self.assertLessEqual(
                             d / current_data_e[x],
-                            0.01,
+                            self.relative_change,
                             f"key {x}. old value: {current_data_e[x]}, new value: {check_fixtures_data_e[x]}\n{self.diff(current_data_e, check_fixtures_data_e)}",
                         )
                     except ZeroDivisionError:
@@ -2057,6 +2060,7 @@ class PipeCheckerRunner:
         only_response_times: bool,
         ignore_order: bool,
         validate_processed_bytes: bool,
+        relative_change: float,
     ) -> PipeChecker:
         return PipeChecker(
             request,
@@ -2066,6 +2070,7 @@ class PipeCheckerRunner:
             only_response_times,
             ignore_order,
             validate_processed_bytes,
+            relative_change,
         )
 
     def _delta_percentage(self, checker: float, current: float) -> float:
@@ -2085,6 +2090,7 @@ class PipeCheckerRunner:
         only_response_times: bool,
         ignore_order: bool,
         validate_processed_bytes: bool,
+        relative_change: float,
         failfast: bool,
         custom_output: bool = False,
         debug: bool = False,
@@ -2123,7 +2129,13 @@ class PipeCheckerRunner:
         for _, request in enumerate(pipe_requests_to_check):
             suite.addTest(
                 self._get_checker(
-                    request, checker_pipe_name, token, only_response_times, ignore_order, validate_processed_bytes
+                    request,
+                    checker_pipe_name,
+                    token,
+                    only_response_times,
+                    ignore_order,
+                    validate_processed_bytes,
+                    relative_change,
                 )
             )
 
@@ -2264,6 +2276,7 @@ async def check_pipe(
     populate: bool,
     cl: TinyB,
     limit: int = 0,
+    relative_change: float = 0.01,
     sample_by_params: int = 0,
     only_response_times=False,
     matches: Optional[List[str]] = None,
@@ -2342,6 +2355,7 @@ async def check_pipe(
         only_response_times,
         ignore_order,
         validate_processed_bytes,
+        relative_change,
         failfast,
     )
 
@@ -2556,6 +2570,7 @@ async def new_pipe(
     run_tests: bool = False,
     as_standard: bool = False,
     tests_to_run: int = 0,
+    tests_relative_change: float = 0.01,
     tests_to_sample_by_params: int = 0,
     tests_filter_by: Optional[List[str]] = None,
     tests_failfast: bool = False,
@@ -2600,6 +2615,7 @@ async def new_pipe(
                         tb_client,
                         only_response_times=only_response_times,
                         limit=tests_to_run,
+                        relative_change=tests_relative_change,
                         sample_by_params=tests_to_sample_by_params,
                         matches=tests_filter_by,
                         failfast=tests_failfast,
@@ -3206,6 +3222,7 @@ async def exec_file(
     run_tests=False,
     as_standard=False,
     tests_to_run: int = 0,
+    tests_relative_change: float = 0.01,
     tests_to_sample_by_params: int = 0,
     tests_filter_by: Optional[List[str]] = None,
     tests_failfast: bool = False,
@@ -3235,6 +3252,7 @@ async def exec_file(
             run_tests=run_tests,
             as_standard=as_standard,
             tests_to_run=tests_to_run,
+            tests_relative_change=tests_relative_change,
             tests_to_sample_by_params=tests_to_sample_by_params,
             tests_filter_by=tests_filter_by,
             tests_failfast=tests_failfast,
@@ -3805,6 +3823,7 @@ async def folder_push(
     raise_on_exists: bool = False,
     verbose: bool = True,
     tests_to_run: int = 0,
+    tests_relative_change: float = 0.01,
     tests_sample_by_params: int = 0,
     tests_filter_by: Optional[List[str]] = None,
     tests_failfast: bool = False,
@@ -3991,6 +4010,7 @@ async def folder_push(
                             run_tests,
                             as_standard,
                             tests_to_run,
+                            tests_relative_change,
                             tests_sample_by_params,
                             tests_filter_by,
                             tests_failfast,

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird/sql_template.py

@@ -42,6 +42,24 @@ class SQLTemplateException(ValueError):
 
 
 DEFAULT_PARAM_NAMES = ["format", "q"]
+RESERVED_PARAM_NAMES = [
+    "__tb__semver",
+    "debug_source_tables",
+    "debug",
+    "explain",
+    "finalize_aggregations",
+    "output_format_json_quote_64bit_integers",
+    "output_format_json_quote_denormals",
+    "output_format_parquet_string_as_string",
+    "pipeline",
+    "playground",
+    "q",
+    "query_id",
+    "release_replacements",
+    "tag",
+    "template_parameters",
+    "token",
+]
 
 parameter_types = [
     "String",
@@ -1771,6 +1789,8 @@ def get_template_and_variables(sql: str, name: Optional[str], escape_arrays: boo
     it is important to NOT MODIFY THESE OBJECTS.
     Neither render_sql_template() or generate() modify them, so neither should you
     """
+    variable_warnings = []
+
     try:
         t = Template(sql, name)
         template_variables = get_var_names(t)
@@ -1780,10 +1800,12 @@ def get_template_and_variables(sql: str, name: Optional[str], escape_arrays: boo
                 name = variable["name"]
                 line = variable["line"]
                 raise ValueError(f'"{name}" can not be used as a variable name, line {line}')
+            if variable["name"] in RESERVED_PARAM_NAMES:
+                variable_warnings.append(variable["name"])
 
         wrap_vars(t, escape_arrays=escape_arrays)
 
-        return t, template_variables
+        return t, template_variables, variable_warnings
     except SecurityException as e:
         raise SQLTemplateException(e)
 
@@ -1836,96 +1858,96 @@ def render_sql_template(
     variables: Optional[dict] = None,
     test_mode: bool = False,
     name: Optional[str] = None,
-) -> Tuple[str, dict]:
+) -> Tuple[str, dict, list]:
     """
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
-    ("select * from table where f = toFloat32('-1.0')", {})
+    ("select * from table where f = toFloat32('-1.0')", {}, [])
     >>> render_sql_template("{% if defined(open) %}ERROR{% else %}YEAH!{% end %}")
-    ('YEAH!', {})
+    ('YEAH!', {}, [])
     >>> render_sql_template("{% if defined(close) %}ERROR{% else %}YEAH!{% end %}")
-    ('YEAH!', {})
+    ('YEAH!', {}, [])
     >>> render_sql_template("{% if defined(input) %}ERROR{% else %}YEAH!{% end %}")
-    ('YEAH!', {})
+    ('YEAH!', {}, [])
     >>> render_sql_template("{% if defined(print) %}ERROR{% else %}YEAH!{% end %}")
-    ('YEAH!', {})
+    ('YEAH!', {}, [])
     >>> render_sql_template("select * from table where str = {{foo}}", { 'foo': 'test' })
-    ("select * from table where str = 'test'", {})
+    ("select * from table where str = 'test'", {}, [])
     >>> render_sql_template("select * from table where f = {{foo}}", { 'foo': 1.0 })
-    ('select * from table where f = 1.0', {})
+    ('select * from table where f = 1.0', {}, [])
     >>> render_sql_template("select {{Boolean(foo)}} from table", { 'foo': True })
-    ('select 1 from table', {})
+    ('select 1 from table', {}, [])
     >>> render_sql_template("select {{Boolean(foo)}} from table", { 'foo': False })
-    ('select 0 from table', {})
+    ('select 0 from table', {}, [])
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': 1 })
-    ("select * from table where f = toFloat32('1.0')", {})
+    ("select * from table where f = toFloat32('1.0')", {}, [])
     >>> render_sql_template("select * from table where f = {{foo}}", { 'foo': "';drop table users;" })
-    ("select * from table where f = '\\\\';drop table users;'", {})
+    ("select * from table where f = '\\\\';drop table users;'", {}, [])
     >>> render_sql_template("select * from {{symbol(foo)}}", { 'foo': 'table-name' })
-    ('select * from `table-name`', {})
+    ('select * from `table-name`', {}, [])
     >>> render_sql_template("select * from {{symbol(foo)}}", { 'foo': '"table-name"' })
-    ('select * from `table-name`', {})
+    ('select * from `table-name`', {}, [])
     >>> render_sql_template("select * from {{table(foo)}}", { 'foo': '"table-name"' })
-    ('select * from table-name', {})
+    ('select * from table-name', {}, [])
     >>> render_sql_template("select * from {{Int32(foo)}}", { 'foo': 'non_int' })
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Error validating 'non_int' to type Int32
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", test_mode=True)
-    ("select * from table where f = toFloat32('0.0')", {})
+    ("select * from table where f = toFloat32('0.0')", {}, [])
     >>> render_sql_template("SELECT * FROM query_log__dev where a = {{test}}", test_mode=True)
-    ("SELECT * FROM query_log__dev where a = '__placeholder__'", {})
+    ("SELECT * FROM query_log__dev where a = '__placeholder__'", {}, [])
     >>> render_sql_template("SELECT {{test}}", {'token':'testing'})
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: expression "test" evaluated to null
     >>> render_sql_template('{% if test %}SELECT 1{% else %} select 2 {% end %}')
-    (' select 2 ', {})
+    (' select 2 ', {}, [])
     >>> render_sql_template('{% if Int32(test, 1) %}SELECT 1{% else %} select 2 {% end %}')
-    ('SELECT 1', {})
+    ('SELECT 1', {}, [])
     >>> render_sql_template('{% for v in test %}SELECT {{v}} {% end %}',test_mode=True)
-    ("SELECT '__placeholder__' SELECT '__placeholder__' SELECT '__placeholder__' ", {})
+    ("SELECT '__placeholder__' SELECT '__placeholder__' SELECT '__placeholder__' ", {}, [])
     >>> render_sql_template("select {{Int32(foo, 1)}}", test_mode=True)
-    ("select toInt32('1')", {})
+    ("select toInt32('1')", {}, [])
     >>> render_sql_template("SELECT count() c FROM test_table where a > {{Float32(myvar)}} {% if defined(my_condition) %} and c = Int32({{my_condition}}){% end %}", {'myvar': 1.0})
-    ("SELECT count() c FROM test_table where a > toFloat32('1.0') ", {})
+    ("SELECT count() c FROM test_table where a > toFloat32('1.0') ", {}, [])
     >>> render_sql_template("SELECT count() c FROM where {{sql_and(a=a, b=b)}}", {'a': '1', 'b': '2'})
-    ("SELECT count() c FROM where a = '1' and b = '2'", {})
+    ("SELECT count() c FROM where a = '1' and b = '2'", {}, [])
     >>> render_sql_template("SELECT count() c FROM where {{sql_and(a=a, b=b)}}", {'b': '2'})
-    ("SELECT count() c FROM where b = '2'", {})
+    ("SELECT count() c FROM where b = '2'", {}, [])
     >>> render_sql_template("SELECT count() c FROM where {{sql_and(a=Int(a, defined=False), b=Int(b, defined=False))}}", {'b': '2'})
-    ('SELECT count() c FROM where b = 2', {})
+    ('SELECT count() c FROM where b = 2', {}, [])
     >>> render_sql_template("SELECT count() c FROM where {{sql_and(a__in=Array(a), b=b)}}", {'a': 'a,b,c','b': '2'})
-    ("SELECT count() c FROM where a in ['a','b','c'] and b = '2'", {})
+    ("SELECT count() c FROM where a in ['a','b','c'] and b = '2'", {}, [])
     >>> render_sql_template("SELECT count() c FROM where {{sql_and(a__not_in=Array(a), b=b)}}", {'a': 'a,b,c','b': '2'})
-    ("SELECT count() c FROM where a not in ['a','b','c'] and b = '2'", {})
+    ("SELECT count() c FROM where a not in ['a','b','c'] and b = '2'", {}, [])
     >>> render_sql_template("SELECT c FROM where a > {{Date(start)}}", test_mode=True)
-    ("SELECT c FROM where a > '2019-01-01'", {})
+    ("SELECT c FROM where a > '2019-01-01'", {}, [])
     >>> render_sql_template("SELECT c FROM where a > {{DateTime(start)}}", test_mode=True)
-    ("SELECT c FROM where a > '2019-01-01 00:00:00'", {})
+    ("SELECT c FROM where a > '2019-01-01 00:00:00'", {}, [])
     >>> render_sql_template("SELECT c FROM where a > {{DateTime(start)}}", {'start': '2018-09-07 23:55:00'})
-    ("SELECT c FROM where a > '2018-09-07 23:55:00'", {})
+    ("SELECT c FROM where a > '2018-09-07 23:55:00'", {}, [])
     >>> render_sql_template('SELECT * FROM tracker {% if defined(start) %} {{DateTime(start)}} and {{DateTime(end)}} {% end %}', {'start': '2019-08-01 00:00:00', 'end': '2019-08-02 00:00:00'})
-    ("SELECT * FROM tracker  '2019-08-01 00:00:00' and '2019-08-02 00:00:00' ", {})
+    ("SELECT * FROM tracker  '2019-08-01 00:00:00' and '2019-08-02 00:00:00' ", {}, [])
     >>> render_sql_template('SELECT * from test limit {{Int(limit)}}', test_mode=True)
-    ('SELECT * from test limit 0', {})
+    ('SELECT * from test limit 0', {}, [])
     >>> render_sql_template('SELECT {{symbol(attr)}} from test', test_mode=True)
-    ('SELECT `placeholder` from test', {})
+    ('SELECT `placeholder` from test', {}, [])
     >>> render_sql_template('SELECT {{Array(foo)}}', {'foo': 'a,b,c,d'})
-    ("SELECT ['a','b','c','d']", {})
+    ("SELECT ['a','b','c','d']", {}, [])
     >>> render_sql_template("SELECT {{Array(foo, 'Int32')}}", {'foo': '1,2,3,4'})
-    ('SELECT [1,2,3,4]', {})
+    ('SELECT [1,2,3,4]', {}, [])
     >>> render_sql_template("SELECT {{Array(foo, 'Int32')}}", test_mode=True)
-    ('SELECT [0,0]', {})
+    ('SELECT [0,0]', {}, [])
     >>> render_sql_template("SELECT {{Array(foo)}}", test_mode=True)
-    ("SELECT ['__placeholder__0','__placeholder__1']", {})
+    ("SELECT ['__placeholder__0','__placeholder__1']", {}, [])
     >>> render_sql_template("{{max_threads(2)}} SELECT 1")
-    ('-- max_threads 2\\n SELECT 1', {'max_threads': 2})
+    ('-- max_threads 2\\n SELECT 1', {'max_threads': 2}, [])
     >>> render_sql_template("SELECT {{String(foo)}}", test_mode=True)
-    ("SELECT '__placeholder__'", {})
+    ("SELECT '__placeholder__'", {}, [])
     >>> render_sql_template("SELECT {{String(foo, 'test')}}", test_mode=True)
-    ("SELECT 'test'", {})
+    ("SELECT 'test'", {}, [])
     >>> render_sql_template("SELECT {{String(foo, 'test')}}", {'foo': 'tt'})
-    ("SELECT 'tt'", {})
+    ("SELECT 'tt'", {}, [])
     >>> render_sql_template("SELECT {{String(format, 'test')}}", {'format': 'tt'})
     Traceback (most recent call last):
     ...
@@ -1943,25 +1965,25 @@ def render_sql_template(
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Missing column() default value, use `column(column_name, 'default_column_name')`
     >>> render_sql_template("SELECT {{column(agg)}}", {'agg': 'foo'})
-    ('SELECT `foo`', {})
+    ('SELECT `foo`', {}, [])
     >>> render_sql_template("SELECT {{column(agg)}}", {'agg': '"foo"'})
-    ('SELECT `foo`', {})
+    ('SELECT `foo`', {}, [])
     >>> render_sql_template('{% if not defined(test) %}error("This is an error"){% end %}', {})
-    ('error("This is an error")', {})
+    ('error("This is an error")', {}, [])
     >>> render_sql_template('{% if not defined(test) %}custom_error({error: "This is an error"}){% end %}', {})
-    ('custom_error({error: "This is an error"})', {})
+    ('custom_error({error: "This is an error"})', {}, [])
     >>> render_sql_template("SELECT {{String(foo + 'abcd')}}", test_mode=True)
-    ("SELECT '__placeholder__'", {})
+    ("SELECT '__placeholder__'", {}, [])
     >>> render_sql_template("SELECT {{columns(agg)}}", {})
     Traceback (most recent call last):
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Missing columns() default value, use `columns(column_names, 'default_column_name')`
     >>> render_sql_template("SELECT {{columns(agg, 'a,b,c')}} FROM table", {})
-    ('SELECT `a`,`b`,`c` FROM table', {})
+    ('SELECT `a`,`b`,`c` FROM table', {}, [])
     >>> render_sql_template("SELECT {{columns(agg, 'a,b,c')}} FROM table", {'agg': 'foo'})
-    ('SELECT `foo` FROM table', {})
+    ('SELECT `foo` FROM table', {}, [])
     >>> render_sql_template("SELECT {{columns('a,b,c')}} FROM table", {})
-    ('SELECT `a`,`b`,`c` FROM table', {})
+    ('SELECT `a`,`b`,`c` FROM table', {}, [])
     >>> render_sql_template("% {% if whatever(passenger_count) %}{% end %}", test_mode=True)
     Traceback (most recent call last):
     ...
@@ -1971,13 +1993,13 @@ def render_sql_template(
     ...
     SyntaxError: invalid syntax
     >>> render_sql_template("SELECT * FROM dim_fecha_evento where foo like {{sql_unescape(String(pepe), '%')}}", {"pepe": 'raul_el_bueno_is_the_best_%'})
-    ("SELECT * FROM dim_fecha_evento where foo like 'raul_el_bueno_is_the_best_%'", {})
+    ("SELECT * FROM dim_fecha_evento where foo like 'raul_el_bueno_is_the_best_%'", {}, [])
     >>> render_sql_template("SELECT * FROM table WHERE field={{String(field_filter)}}", {"field_filter": 'action."test run"'})
-    ('SELECT * FROM table WHERE field=\\'action.\\\\"test run\\\\"\\'', {})
+    ('SELECT * FROM table WHERE field=\\'action.\\\\"test run\\\\"\\'', {}, [])
     >>> render_sql_template("SELECT {{Int128(foo)}} as x, {{Int128(bar)}} as y", {'foo': -170141183460469231731687303715884105728, 'bar': 170141183460469231731687303715884105727})
-    ("SELECT toInt128('-170141183460469231731687303715884105728') as x, toInt128('170141183460469231731687303715884105727') as y", {})
+    ("SELECT toInt128('-170141183460469231731687303715884105728') as x, toInt128('170141183460469231731687303715884105727') as y", {}, [])
     >>> render_sql_template("SELECT {{Int256(foo)}} as x, {{Int256(bar)}} as y", {'foo': -57896044618658097711785492504343953926634992332820282019728792003956564819968, 'bar': 57896044618658097711785492504343953926634992332820282019728792003956564819967})
-    ("SELECT toInt256('-57896044618658097711785492504343953926634992332820282019728792003956564819968') as x, toInt256('57896044618658097711785492504343953926634992332820282019728792003956564819967') as y", {})
+    ("SELECT toInt256('-57896044618658097711785492504343953926634992332820282019728792003956564819968') as x, toInt256('57896044618658097711785492504343953926634992332820282019728792003956564819967') as y", {}, [])
     >>> render_sql_template('% SELECT * FROM {% import os %}{{ os.popen("whoami").read() }}')
     Traceback (most recent call last):
     ...
@@ -1999,21 +2021,21 @@ def render_sql_template(
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Invalid BinOp: Pow()
     >>> render_sql_template("% SELECT {{Array(click_selector, 'String', 'pre,pro')}}")
-    ("% SELECT ['pre','pro']", {})
+    ("% SELECT ['pre','pro']", {}, [])
     >>> render_sql_template("% SELECT {{Array(click_selector, 'String', 'pre,pro')}}", {'click_selector': 'hi,hello'})
-    ("% SELECT ['hi','hello']", {})
+    ("% SELECT ['hi','hello']", {}, [])
     >>> render_sql_template("% SELECT now() > {{DateTime64(variable, '2020-09-09 10:10:10.000')}}", {})
-    ("% SELECT now() > '2020-09-09 10:10:10.000'", {})
+    ("% SELECT now() > '2020-09-09 10:10:10.000'", {}, [])
     >>> render_sql_template("% SELECT {% if defined(x) %} x, 1", {})
     Traceback (most recent call last):
     ...
     tinybird.tornado_template.UnClosedIfError: Missing {% end %} block for if at line 1
     >>> render_sql_template("% SELECT * FROM employees WHERE 0 {% for kv in JSON(payload) %} OR department = {{kv['dp']}} {% end %}")
-    ('% SELECT * FROM employees WHERE 0 ', {})
+    ('% SELECT * FROM employees WHERE 0 ', {}, [])
     >>> render_sql_template("% SELECT * FROM employees WHERE 0 {% for kv in JSON(payload, '[{\\"dp\\":\\"Sales\\"}]') %} OR department = {{kv['dp']}} {% end %}")
-    ("% SELECT * FROM employees WHERE 0  OR department = 'Sales' ", {})
+    ("% SELECT * FROM employees WHERE 0  OR department = 'Sales' ", {}, [])
     >>> render_sql_template("% SELECT * FROM employees WHERE 0 {% for kv in JSON(payload) %} OR department = {{kv['dp']}} {% end %}", { 'payload': '[{"dp":"Design"},{"dp":"Marketing"}]'})
-    ("% SELECT * FROM employees WHERE 0  OR department = 'Design'  OR department = 'Marketing' ", {})
+    ("% SELECT * FROM employees WHERE 0  OR department = 'Design'  OR department = 'Marketing' ", {}, [])
     >>> render_sql_template("% {% for kv in JSON(payload) %} department = {{kv['dp']}} {% end %}", test_mode=True)
     Traceback (most recent call last):
     ...
@@ -2023,22 +2045,26 @@ def render_sql_template(
     ...
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Error parsing JSON: '' - Expecting value: line 1 column 1 (char 0)
     >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": "1,2"})
-    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {}, [])
     >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": ["1","2"]})
-    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {}, [])
     >>> render_sql_template("% {% if defined(test) %}{% set _total = sum(test) %} SELECT test as aa, {{Array(test, 'Int32')}} as test, {{_total}} as a {% end %}", {"test": "1,2"})
-    ('%  SELECT test as aa, [1,2] as test, 3 as a ', {})
+    ('%  SELECT test as aa, [1,2] as test, 3 as a ', {}, [])
     >>> render_sql_template("% {% if defined(test) %}{% set _groupByCSV = ','.join(test) %} SELECT test as aa, {{Array(test, 'String')}} as test, {{_groupByCSV}} as a {% end %}", {"test": ["1","2"]})
-    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {})
+    ("%  SELECT test as aa, ['1','2'] as test, '1,2' as a ", {}, [])
     >>> render_sql_template("% SELECT {% if defined(x) %} x, 1")
     Traceback (most recent call last):
     ...
     tinybird.tornado_template.UnClosedIfError: Missing {% end %} block for if at line 1
+    >>> render_sql_template("select * from table where str = {{pipeline}}", { 'pipeline': 'test' })
+    ("select * from table where str = 'test'", {}, ['pipeline'])
     """
     escape_split_to_array = ff_split_to_array_escape.get(False)
     bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
 
-    t, template_variables = get_template_and_variables(sql, name, escape_arrays=escape_split_to_array)
+    t, template_variables, variable_warnings = get_template_and_variables(
+        sql, name, escape_arrays=escape_split_to_array
+    )
 
     if not bypass_preprocess_variables and variables is not None:
         processed_variables = preprocess_variables(variables, t)
@@ -2070,7 +2096,7 @@ def render_sql_template(
         v.update(type_fns)
 
     try:
-        return generate(t, **v)
+        return *generate(t, **v), variable_warnings
     except NameError as e:
         raise SQLTemplateException(e, documentation="/cli/advanced-templates.html#defined")
     except SQLTemplateException:

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird/tb_cli_modules/pipe.py

@@ -539,6 +539,12 @@ async def print_pipe(ctx: Context, pipe: str, query: str, format_: str):
     help="When set, the checker will get Main Workspace requests",
     hidden=True,
 )
+@click.option(
+    "--relative-change",
+    type=float,
+    default=0.01,
+    help="When set, the checker will validate the new version has less than this distance with the current version",
+)
 @click.pass_context
 @coro
 async def regression_test(
@@ -556,6 +562,7 @@ async def regression_test(
     ignore_order: bool,
     validate_processed_bytes: bool,
     check_requests_from_main: bool,
+    relative_change: float,
 ):
     """
     Run regression tests on Tinybird
@@ -583,6 +590,7 @@ async def regression_test(
         no_versions=no_versions,
         run_tests=True,
         tests_to_run=limit,
+        tests_relative_change=relative_change,
         tests_sample_by_params=sample_by_params,
         tests_filter_by=match,
         tests_failfast=failfast,

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird/tornado_template.py

@@ -892,7 +892,15 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
             reader.consume(2)
             if not contents:
                 reader.raise_parse_error("Empty expression")
-            body.chunks.append(_Expression(contents, line))
+
+            try:
+                body.chunks.append(_Expression(contents, line))
+            except SyntaxError:
+                operator, _, _ = contents.partition(" ")
+
+                if "from" in operator:
+                    reader.raise_parse_error('"from" is a forbidden word')
+                raise
             continue
 
         # Block

{tinybird-cli-4.1.1.dev1 → tinybird-cli-5.0.0.dev0}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 4.1.1.dev1
+Version: 5.0.0.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,7 +18,18 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
-4.1.1.dev0
+5.0.0.dev0
+************
+
+- `Changed` Make `insertion_date`` column explicit. This column is no longer inferred, it must be present in the Data Source file.
+
+4.1.2.dev0
+************
+
+- `Added` parameter to `tb pipe regression-test` --relative-change
+
+
+4.1.1
 ************
 
 - `Fixed` workspace info gathering when switching branches. We were prioritizing the general workspace properties over the user ones.
@@ -27,7 +38,7 @@ Changelog
 ************
 
 - `Added` `tb token create` command to be able to create static and JWT tokens from the CLI. You can check more information at https://www.tinybird.co/blog-posts/jwt-api-endpoints-public-beta
-- `Fixed` `tb init --git` to pin `tinybird-cli>=4,<5` in `requirements.txt` to avoid issues with the latest version of the CLI. 
+- `Fixed` `tb init --git` to pin `tinybird-cli>=4,<5` in `requirements.txt` to avoid issues with the latest version of the CLI.
 
 4.0.0
 ************
@@ -85,7 +96,7 @@ This is a major release, please read the commands affected below and consider up
 ************
 
 - `Changed` fixed major version of tinybird-cli to lower than 4 when using `tb init --git`
-- `Changed` behavior when running `tb deploy` on a branch to push the connection settings to the backend. This change is the backend that decides what to do. 
+- `Changed` behavior when running `tb deploy` on a branch to push the connection settings to the backend. This change is the backend that decides what to do.
 - `Changed` `tb pipe sink` commands are now available
 - `Fixed` regression tests query when filtering by specific parameter
 - `Fixed` Avoid `tb fmt` to error if there's a `CASE` in the sql