tinybird-cli 5.13.1.dev0__tar.gz → 5.13.1.dev2__tar.gz

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.13.1.dev0
+Version: 5.13.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,10 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.13.1.dev2
+***********
+
+- `Added` support for `EXPORT_WRITE_STRATEGY` parameter for sinks, in `.pipe` files.
+
+5.13.1.dev1
+***********
+
+- `Changed` environment variables are now evaluated in the SQL nodes.
+
 5.13.0
 ***********
 
-- `Addded` additional metadata to `tb branch ls` command
+- `Added` additional metadata to `tb branch ls` command
 - `Changed` dependencies to be less restrictive:
     - cryptography: from ``>=41.0.0`` to ``~=41.0.0``
     - snowflake-connector-python: from ``==3.12.3`` to ``~=3.12.3``

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '5.13.1.dev0'
-__revision__ = '086d6df'
+__version__ = '5.13.1.dev2'
+__revision__ = '5103f8f'

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/client.py

@@ -80,7 +80,7 @@ def parse_error_response(response: Response) -> str:
         return f"Server error, cannot parse response. {response.content.decode('utf-8')}"
 
 
-class TinyB(object):
+class TinyB:
     MAX_GET_LENGTH = 4096
 
     def __init__(

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/datafile.py

@@ -212,6 +212,7 @@ class ExportReplacements:
         ("export_file_template", "file_template", None),
         ("export_format", "format", "csv"),
         ("export_compression", "compression", None),
+        ("export_write_strategy", "write_strategy", None),
         ("export_strategy", "strategy", "@new"),
         ("export_kafka_topic", "kafka_topic", None),
         ("kafka_connection_name", "connection", None),
@@ -1051,9 +1052,9 @@ def parse(
         def _f(sql: str, **kwargs: Any) -> None:
             if not parser_state.current_node:
                 raise ParseException("SQL must be called after a NODE command")
-            parser_state.current_node[var_name] = (
-                textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
-            )
+            _sql = textwrap.dedent(sql).rstrip() if "%" not in sql.strip()[0] else sql.strip()
+            _sql = eval_var(_sql)
+            parser_state.current_node[var_name] = _sql
 
         # HACK this cast is needed because Mypy
         return cast(Callable[[str, KwArg(Any)], None], _f)
@@ -1239,6 +1240,7 @@ def parse(
         "export_format": assign_var("export_format"),
         "export_strategy": assign_var("export_strategy"),
         "export_compression": assign_var("export_compression"),
+        "export_write_strategy": assign_var("export_write_strategy"),
         "export_kafka_topic": assign_var("export_kafka_topic"),
     }
 
@@ -2255,16 +2257,16 @@ class PipeCheckerRunner:
         class PipeCheckerTextTestResult(unittest.TextTestResult):
             def __init__(self, *args: Any, **kwargs: Any) -> None:
                 self.custom_output = kwargs.pop("custom_output", False)
-                super(PipeCheckerTextTestResult, self).__init__(*args, **kwargs)
+                super().__init__(*args, **kwargs)
                 self.success: List[PipeChecker] = []
 
             def addSuccess(self, test: PipeChecker):  # type: ignore
-                super(PipeCheckerTextTestResult, self).addSuccess(test)
+                super().addSuccess(test)
                 self.success.append(test)
 
             def startTest(self, test):
                 if not self.custom_output:
-                    super(PipeCheckerTextTestResult, self).startTest(test)
+                    super().startTest(test)
                 else:
                     super(unittest.TextTestResult, self).startTest(test)
 
@@ -3352,7 +3354,12 @@ async def new_ds(
             existing.sort(key=lambda x: x["name"])
             if len(existing) != len(new) or any([(d, d2) for d, d2 in zip(new, existing) if d != d2]):
                 new_indices = ds.get("params", {}).get("indexes") or "0"
-        if new_description or new_schema or new_ttl or (new_indices is not None) and (not fork_downstream or not fork):
+        if (
+            new_description
+            or new_schema
+            or new_ttl
+            or ((new_indices is not None) and (not fork_downstream or not fork))
+        ):
             alter_response = await client.alter_datasource(
                 ds_name,
                 new_schema=new_schema,
@@ -4866,7 +4873,7 @@ async def format_datasource(
         format_tags(file_parts, doc)
         format_schema(file_parts, doc.nodes[0])
         format_indices(file_parts, doc.nodes[0])
-        await format_engine(file_parts, doc.nodes[0], only_ttl=True if not for_deploy_diff else False, client=client)
+        await format_engine(file_parts, doc.nodes[0], only_ttl=bool(not for_deploy_diff), client=client)
         if for_deploy_diff:
             format_import_settings(file_parts, doc.nodes[0])
         format_shared_with(file_parts, doc)
@@ -5136,7 +5143,7 @@ async def format_pipe(
             doc.includes,
             line_length=line_length,
             unroll_includes=unroll_includes,
-            lower_keywords=True if for_deploy_diff else False,
+            lower_keywords=bool(for_deploy_diff),
         )
 
     if not unroll_includes:

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/datatypes.py

@@ -429,6 +429,6 @@ def get_decimal_limits(p, s):
     (Decimal('-99999999999999999999999999999999999999.99999999999999999999999999999999999999'), Decimal('99999999999999999999999999999999999999.99999999999999999999999999999999999999'))
     """
     with decimal.localcontext(prec=p + 2):
-        max_value = Decimal(((10**p) - 1)) / (10**s)
+        max_value = Decimal((10**p) - 1) / (10**s)
         min_value = -max_value
     return min_value, max_value

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/sql.py

@@ -605,11 +605,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
             context = context_stack[-1]
             c = schema[i]
 
-            if context == "'" and c == "'":
-                context_stack.pop()
-            elif context == '"' and c == '"':
-                context_stack.pop()
-            elif context == "(" and c == ")":
+            if (context == "'" and c == "'") or (context == '"' and c == '"') or (context == "(" and c == ")"):
                 context_stack.pop()
             elif c == "'" and (context is None or context == "("):
                 context_stack.append("'")
@@ -619,11 +615,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
                 context_stack.append("(")
             elif context is None and lookahead_matches(lookup):
                 return schema[begin:i].strip(" \t\r\n")
-            elif context is None and c not in valid_chars_fn:
-                raise ValueError(
-                    format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
-                )
-            elif context == "(" and c not in valid_chars_fn:
+            elif (context is None and c not in valid_chars_fn) or (context == "(" and c not in valid_chars_fn):
                 raise ValueError(
                     format_parse_error(schema, i, pos, "wrong value, please check the schema syntax", line=line)
                 )
@@ -738,9 +730,7 @@ def _parse_table_structure(schema: str) -> List[Dict[str, Any]]:
             advance(",")
             valid_next = []
             add_column("COMMA")
-        elif found == NEW_LINE:
-            i += 1
-        elif name == "INDEX" and not found:
+        elif found == NEW_LINE or (name == "INDEX" and not found):
             i += 1
         else:
             raise ValueError(

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/sql_template.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from functools import lru_cache
 from io import StringIO
 from json import loads
-from typing import Any, List, Optional, Tuple, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
@@ -33,7 +33,7 @@ def is_secret_template_key(key: str) -> bool:
 
 class TemplateExecutionResults(dict):
     def __init__(self, *args, **kwargs):
-        super(TemplateExecutionResults, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.template_params = set()
         self.ch_params = set()
 
@@ -1383,6 +1383,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     for key in kwargs.get("tb_secrets", []):
         if is_secret_template_key(key):
             template_execution_results.add_template_param(key)
+
     if TB_SECRET_IN_TEST_MODE in kwargs:
         template_execution_results[TB_SECRET_IN_TEST_MODE] = None
 
@@ -1493,7 +1494,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
         raise e
 
 
-class CodeWriter(object):
+class CodeWriter:
     def __init__(self, file, template):
         self.file = file
         self.current_template = template
@@ -1504,7 +1505,7 @@ class CodeWriter(object):
         return self._indent
 
     def indent(self):
-        class Indenter(object):
+        class Indenter:
             def __enter__(_):
                 self._indent += 1
                 return self
@@ -1989,6 +1990,7 @@ def render_sql_template(
     secrets: Optional[List[str]] = None,
     test_mode: bool = False,
     name: Optional[str] = None,
+    local_variables: Optional[dict] = None,
 ) -> Tuple[str, TemplateExecutionResults, list]:
     """
     >>> render_sql_template("select * from table where f = {{Float32(foo)}}", { 'foo': -1 })
@@ -2260,6 +2262,9 @@ def render_sql_template(
             }
         )
 
+        if local_variables:
+            v.update(local_variables)
+
     else:
         v = {x["name"]: None for x in template_variables}
         if variables:
@@ -2267,8 +2272,12 @@ def render_sql_template(
 
         if secrets:
             v.update({"tb_secrets": secrets})
+
         v.update(type_fns)
 
+        if local_variables:
+            v.update(local_variables)
+
     try:
         sql, template_execution_results = generate(t, **v)
         try:
@@ -2298,3 +2307,37 @@ def render_sql_template(
                 documentation="/cli/advanced-templates.html",
             )
         raise e
+
+
+def extract_variables_from_sql(sql: str, params: List[Dict[str, Any]]) -> Dict[str, Any]:
+    sql = sql[1:] if sql[0] == "%" else sql
+    defaults = {}
+    mock_data = {}
+    try:
+        for param in params:
+            mock_data[param["name"]] = "__NO__VALUE__DEFINED__"
+        # Initialize a dictionary to track variables
+        variable_tracker = {}
+
+        # Wrapper function to capture variable assignments
+        def capture_variable(name, value):
+            variable_tracker[name] = value
+            return value
+
+        # Modify the template by adding capture hooks
+        tracked_template_string = sql
+        for var_name in mock_data.keys():
+            tracked_template_string += f"{{% set __ = capture_variable('{var_name}', {var_name}) %}}"
+
+        # Define the modified template with tracking
+        template = Template(tracked_template_string)
+        type_fns = get_transform_types()
+        template.generate(**mock_data, **type_fns, capture_variable=capture_variable)
+        for var_name, value in variable_tracker.items():
+            if value != "__NO__VALUE__DEFINED__":
+                defaults[var_name] = value
+    except Exception as e:
+        logging.error(f"Error extracting variables from sql: {e}")
+        return {}
+
+    return defaults

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/tb_cli_modules/cli.py

@@ -1128,7 +1128,7 @@ async def materialize(
             click.echo(FeedbackManager.info_materialize_push_pipe_override(name=f_pipe.name))
             option = click.prompt(FeedbackManager.prompt_choose(), default=1)
         force = True
-        check = True if option == 1 else False
+        check = option == 1
 
         filename = str(f_pipe.absolute())
         to_run = await folder_push(

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird/tornado_template.py

@@ -242,7 +242,7 @@ def filter_whitespace(mode, text):
         raise Exception("invalid whitespace mode %s" % mode)
 
 
-class Template(object):
+class Template:
     """A compiled template.
 
     We compile into Python from the given template_string. You can generate
@@ -378,7 +378,7 @@ class Template(object):
         return ancestors
 
 
-class BaseLoader(object):
+class BaseLoader:
     """Base class for template loaders.
 
     You must use a template loader to use template constructs like
@@ -438,7 +438,7 @@ class Loader(BaseLoader):
     """A template loader that loads from a single root directory."""
 
     def __init__(self, root_directory, **kwargs):
-        super(Loader, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.root = os.path.abspath(root_directory)
 
     def resolve_path(self, name, parent_path=None):
@@ -466,7 +466,7 @@ class DictLoader(BaseLoader):
     """A template loader that loads from a dictionary."""
 
     def __init__(self, dict, **kwargs):
-        super(DictLoader, self).__init__(**kwargs)
+        super().__init__(**kwargs)
         self.dict = dict
 
     def resolve_path(self, name, parent_path=None):
@@ -484,7 +484,7 @@ class DictLoader(BaseLoader):
         return Template(self.dict[name], name=name, loader=self)
 
 
-class _Node(object):
+class _Node:
     def each_child(self):
         return ()
 
@@ -646,7 +646,7 @@ class _Expression(_Node):
 
 class _Module(_Expression):
     def __init__(self, expression, line):
-        super(_Module, self).__init__("_tt_modules." + expression, line, raw=True)
+        super().__init__("_tt_modules." + expression, line, raw=True)
 
 
 class _Text(_Node):
@@ -713,7 +713,7 @@ class ParseError(Exception):
         return "%s at %s:%d" % (self.message, self.filename, self.lineno)
 
 
-class _CodeWriter(object):
+class _CodeWriter:
     def __init__(self, file, named_blocks, loader, current_template):
         self.file = file
         self.named_blocks = named_blocks
@@ -727,7 +727,7 @@ class _CodeWriter(object):
         return self._indent
 
     def indent(self):
-        class Indenter(object):
+        class Indenter:
             def __enter__(_):
                 self._indent += 1
                 return self
@@ -742,7 +742,7 @@ class _CodeWriter(object):
         self.include_stack.append((self.current_template, line))
         self.current_template = template
 
-        class IncludeTemplate(object):
+        class IncludeTemplate:
             def __enter__(_):
                 return self
 
@@ -761,7 +761,7 @@ class _CodeWriter(object):
         print("    " * indent + line + line_comment, file=self.file)
 
 
-class _TemplateReader(object):
+class _TemplateReader:
     def __init__(self, name, text, whitespace):
         self.name = name
         self.text = text
@@ -1169,9 +1169,7 @@ def check_valid_expr(expr):
                 check_valid_expr(expr.slice.lower)
             if expr.slice.upper is not None:
                 check_valid_expr(expr.slice.upper)
-        elif isinstance(expr.slice, ast.Constant):
-            check_valid_expr(expr.slice)
-        elif isinstance(expr.slice, ast.Subscript):
+        elif isinstance(expr.slice, ast.Constant) or isinstance(expr.slice, ast.Subscript):
             check_valid_expr(expr.slice)
         else:
             raise SecurityException(f"Invalid Slice expression: {ast.dump(expr.slice)}")
@@ -1180,13 +1178,7 @@ def check_valid_expr(expr):
             check_valid_expr(key)
         for value in expr.values:
             check_valid_expr(value)
-    elif isinstance(expr, ast.Tuple):
-        for x in expr.elts:
-            check_valid_expr(x)
-    elif isinstance(expr, ast.List):
-        for x in expr.elts:
-            check_valid_expr(x)
-    elif isinstance(expr, ast.Set):
+    elif isinstance(expr, ast.Tuple) or isinstance(expr, ast.List) or isinstance(expr, ast.Set):
         for x in expr.elts:
             check_valid_expr(x)
     elif isinstance(expr, ast.JoinedStr):

{tinybird-cli-5.13.1.dev0 → tinybird-cli-5.13.1.dev2}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 5.13.1.dev0
+Version: 5.13.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -18,10 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+5.13.1.dev2
+***********
+
+- `Added` support for `EXPORT_WRITE_STRATEGY` parameter for sinks, in `.pipe` files.
+
+5.13.1.dev1
+***********
+
+- `Changed` environment variables are now evaluated in the SQL nodes.
+
 5.13.0
 ***********
 
-- `Addded` additional metadata to `tb branch ls` command
+- `Added` additional metadata to `tb branch ls` command
 - `Changed` dependencies to be less restrictive:
     - cryptography: from ``>=41.0.0`` to ``~=41.0.0``
     - snowflake-connector-python: from ``==3.12.3`` to ``~=3.12.3``