tinybird 0.0.1.dev67__py3-none-any.whl → 0.0.1.dev69__py3-none-any.whl

tinybird/ch_utils/engine.py

@@ -456,7 +456,7 @@ ENABLED_ENGINES = [
         MERGETREE_OPTIONS,
     ),
     # AggregatingMergeTree()
-    engine_config("AggregatingMergeTree", options=REPLACINGMERGETREE_OPTIONS),
+    engine_config("AggregatingMergeTree", options=MERGETREE_OPTIONS),
     # CollapsingMergeTree(sign)
     engine_config(
         "CollapsingMergeTree",
@@ -631,9 +631,7 @@ def engine_full_from_dict(
 
     >>> schema = 'sign_column Int8'
     >>> engine_full_from_dict('AggregatingMergeTree', {}, schema=schema)
-    Traceback (most recent call last):
-    ...
-    ValueError: Missing required option 'sorting_key'
+    'AggregatingMergeTree() ORDER BY (tuple())'
 
     >>> columns=[]
     >>> columns.append({'name': 'key_column', 'type': 'Int8', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'key_column'})

tinybird/context.py

@@ -20,5 +20,4 @@ engine: ContextVar[str] = ContextVar("engine")
 wait_parameter: ContextVar[bool] = ContextVar("wait_parameter")
 api_host: ContextVar[str] = ContextVar("api_host")
 ff_split_to_array_escape: ContextVar[bool] = ContextVar("ff_split_to_array_escape")
-ff_preprocess_parameters_circuit_breaker: ContextVar[bool] = ContextVar("ff_preprocess_parameters_circuit_breaker")
 ff_column_json_backticks_circuit_breaker: ContextVar[bool] = ContextVar("ff_column_json_backticks_circuit_breaker")

tinybird/prompts.py

@@ -699,8 +699,8 @@ pipe_instructions = """
     - Nodes can't have the same exact name as the Pipe they belong to.
     - Avoid more than one node per pipe unless it is really necessary or requested by the user.
     - No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
-    - Allowed TYPE values are: endpoint, copy, materialized, sink
-    - Add always the output node in the TYPE section.
+    - Allowed TYPE values are: endpoint, copy, materialized.
+    - Add always the output node in the TYPE section or in the last node of the pipe.
 </pipe_file_instructions>
 """
 
@@ -788,7 +788,7 @@ When you need to work with resources or data in cloud, add always the --cloud fl
 ├── endpoints
 ├── fixtures
 ├── materializations
-├── sinks
+├── pipes
 └── tests
 - The local development server will be available at http://localhost:80. Even if some response uses another base url, use always http://localhost:80.
 - After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.

tinybird/sql_template.py

@@ -14,7 +14,6 @@ from tornado.util import ObjectDict, exec_in, unicode_type
 
 from tinybird.context import (
     ff_column_json_backticks_circuit_breaker,
-    ff_preprocess_parameters_circuit_breaker,
     ff_split_to_array_escape,
 )
 
@@ -2244,14 +2243,13 @@ def render_sql_template(
     tinybird.sql_template.SQLTemplateException: Template Syntax Error: Required parameter is not defined. Check the parameters test. Please provide a value or set a default value in the pipe code.
     """
     escape_split_to_array = ff_split_to_array_escape.get(False)
-    bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
 
     t, template_variables, variable_warnings = get_template_and_variables(
         sql, name, escape_arrays=escape_split_to_array
     )
     template_variables_with_types = get_var_names_and_types_cached(t)
 
-    if not bypass_preprocess_variables and variables is not None:
+    if variables is not None:
         processed_variables = preprocess_variables(variables, template_variables_with_types)
         variables.update(processed_variables)
 

tinybird/sql_toolset.py

@@ -3,7 +3,7 @@ import logging
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
-from typing import FrozenSet, List, Optional, Set, Tuple
+from typing import FrozenSet, List, Optional, Set, Tuple, Union
 
 from chtoolset import query as chquery
 from toposort import toposort
@@ -172,7 +172,7 @@ def tables_or_sql(replacement: dict, table_functions=False) -> set:
         return {replacement}
 
 
-def _separate_as_tuple_if_contains_database_and_table(definition: str) -> str | Tuple[str, str]:
+def _separate_as_tuple_if_contains_database_and_table(definition: str) -> Union[str, Tuple[str, str]]:
     if "." in definition:
         database_and_table_separated = definition.split(".")
         return database_and_table_separated[0], database_and_table_separated[1]
@@ -255,7 +255,7 @@ def replace_tables(
         function_allow_list=function_allow_list,
     )
     seen_tables = set()
-    table: Tuple[str, str] | Tuple[str, str, str]
+    table: Union[Tuple[str, str], Tuple[str, str, str]]
     if function_allow_list is None:
         _enabled_table_functions = ENABLED_TABLE_FUNCTIONS
     else:

tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev67'
-__revision__ = '76a69c0'
+__version__ = '0.0.1.dev69'
+__revision__ = 'f90d907'

tinybird/tb/modules/auth.py

@@ -30,7 +30,7 @@ from tinybird.tb.modules.regions import Region
 @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
 @click.option(
     "--host",
-    help="Set custom host if it's different than https://api.tinybird.co. Check https://www.tinybird.co/docs/api-reference/overview#regions-and-endpoints for the available list of regions",
+    help="Set custom host if it's different than https://api.tinybird.co. Check https://www.tinybird.co/docs/api-reference#regions-and-endpoints for the available list of regions",
 )
 @click.option(
     "--region", envvar="TB_REGION", help="Set region. Run 'tb auth ls' to show available regions. Overrides host."

tinybird/tb/modules/cli.py

@@ -147,12 +147,12 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
 @click.option("--no-deps", is_flag=True, default=False, help="Print only data sources with no pipes using them")
 @click.option("--match", default=None, help="Retrieve any resource matching the pattern")
 @click.option("--pipe", default=None, help="Retrieve any resource used by pipe")
-@click.option("--datasource", default=None, help="Retrieve resources depending on this Data Source")
+@click.option("--datasource", default=None, help="Retrieve resources depending on this data source")
 @click.option(
     "--check-for-partial-replace",
     is_flag=True,
     default=False,
-    help="Retrieve dependant Data Sources that will have their data replaced if a partial replace is executed in the Data Source selected",
+    help="Retrieve dependant data sources that will have their data replaced if a partial replace is executed in the data source selected",
 )
 @click.option("--recursive", is_flag=True, default=False, help="Calculate recursive dependencies")
 @click.pass_context
@@ -186,7 +186,7 @@ async def dependencies(
 
 @cli.command(
     name="diff",
-    short_help="Diffs local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
+    short_help="Diff local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
 )
 @click.argument("filename", type=click.Path(exists=True), nargs=-1, required=False)
 @click.option(
@@ -203,7 +203,7 @@ async def dependencies(
     "--main",
     is_flag=True,
     default=False,
-    help="Diffs local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on a Branch.",
+    help="Diff local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on a Branch.",
     hidden=True,
 )
 @click.pass_context
@@ -255,7 +255,7 @@ async def diff(
 @cli.command()
 @click.argument("query", required=False)
 @click.option("--rows_limit", default=100, help="Max number of rows retrieved")
-@click.option("--pipeline", default=None, help="The name of the Pipe to run the SQL Query")
+@click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
 @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
 @click.option("--node", default=None, help="The NODE name")
 @click.option(

tinybird/tb/modules/common.py

@@ -571,7 +571,7 @@ def region_from_host(region_name_or_host, regions):
 
 def ask_for_user_token(action: str, ui_host: str) -> str:
     return click.prompt(
-        f'\nUse the token called "user token" in order to {action}. Copy it from {ui_host}/tokens and paste it here',
+        f'\nUse the token called "user token" to {action}. Copy it from {ui_host}/tokens and paste it here',
         hide_input=True,
         show_default=False,
         default=None,
@@ -591,13 +591,13 @@ async def check_user_token(ctx: Context, token: str):
     if not is_authenticated.get("is_valid", False):
         raise CLIWorkspaceException(
             FeedbackManager.error_exception(
-                error='Invalid token. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
+                error='Invalid token. Make sure you are using the "user token" instead of the "admin your@email" token.'
             )
         )
     if is_authenticated.get("is_valid") and not is_authenticated.get("is_user", False):
         raise CLIWorkspaceException(
             FeedbackManager.error_exception(
-                error='Invalid user authentication. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
+                error='Invalid user authentication. Make sure you are using the "user token" instead of the "admin your@email" token.'
             )
         )
 
@@ -614,13 +614,13 @@ async def check_user_token_with_client(client: TinyB, token: str):
     if not is_authenticated.get("is_valid", False):
         raise CLIWorkspaceException(
             FeedbackManager.error_exception(
-                error='Invalid token. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
+                error='Invalid token. Make sure you are using the "user token" instead of the "admin your@email" token.'
             )
         )
     if is_authenticated.get("is_valid") and not is_authenticated.get("is_user", False):
         raise CLIWorkspaceException(
             FeedbackManager.error_exception(
-                error='Invalid user authentication. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
+                error='Invalid user authentication. Make sure you are using the "user token" instead of the "admin your@email" token.'
             )
         )
 
@@ -1153,17 +1153,17 @@ def validate_kafka_bootstrap_servers(host_and_port):
 
 def validate_kafka_key(s):
     if not isinstance(s, str):
-        raise CLIException("Key format is not correct, it should be a string")
+        raise CLIException("Key format is not correct, it must be a string")
 
 
 def validate_kafka_secret(s):
     if not isinstance(s, str):
-        raise CLIException("Password format is not correct, it should be a string")
+        raise CLIException("Password format is not correct, it must be a string")
 
 
 def validate_string_connector_param(param, s):
     if not isinstance(s, str):
-        raise CLIConnectionException(param + " format is not correct, it should be a string")
+        raise CLIConnectionException(param + " format is not correct, it must be a string")
 
 
 async def validate_connection_name(client, connection_name, service):
@@ -1434,7 +1434,7 @@ async def try_update_config_with_remote(
 def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
     return (
         click.prompt(
-            f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
+            f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from the .tinyb file" if default_token else ""}',
             hide_input=True,
             show_default=False,
             default=default_token,

tinybird/tb/modules/create.py

@@ -35,17 +35,9 @@ from tinybird.tb.modules.project import Project
 )
 @click.option("--rows", type=int, default=10, help="Number of events to send")
 @click.option("--source", type=str, default="tb", help="Source of the command")
-@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated resources")
 @click.pass_context
 @coro
-async def create(
-    ctx: click.Context,
-    data: Optional[str],
-    prompt: Optional[str],
-    rows: int,
-    source: str,
-    skip: bool,
-) -> None:
+async def create(ctx: click.Context, data: Optional[str], prompt: Optional[str], rows: int, source: str) -> None:
     """Initialize a new project."""
     project: Project = ctx.ensure_object(dict)["project"]
     local_client: TinyB = ctx.ensure_object(dict)["client"]
@@ -80,7 +72,7 @@ async def create(
         result = ""
         if data or prompt:
             click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-            result = await create_resources(local_client, tb_client, user_token, data, prompt, folder, skip)
+            result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
             click.echo(FeedbackManager.success(message="✓ Done!\n"))
 
         if not already_has_cicd(folder):
@@ -125,7 +117,7 @@ async def create(
         click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
 
 
-PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "sinks", "fixtures", "tests")
+PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests")
 
 
 def validate_project_structure(folder: str) -> bool:
@@ -166,7 +158,6 @@ async def create_resources(
     data: Optional[str],
     prompt: Optional[str],
     folder: str,
-    skip: bool,
 ):
     result = ""
     folder_path = Path(folder)
@@ -215,73 +206,39 @@ TYPE ENDPOINT
             ]
         )
         llm = LLM(user_token=user_token, host=tb_client.host)
-        result = ""
-        iterations = 0
-        history = ""
-        generated_paths: list[Path] = []
-
-        while iterations < 10:
-            feedback = ""
-            if iterations > 0:
-                feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
-            if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
-                break
-            else:
-                if iterations > 0:
-                    click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
-                    for path in generated_paths:
-                        path.unlink()
-                    generated_paths = []
-
-                save_context(prompt, feedback)
-                result = llm.ask(system_prompt=create_prompt(resources_xml, feedback, history), prompt=prompt)
-                result = extract_xml(result, "response")
-                history = (
-                    history
-                    + f"""
-                <result_iteration_{iterations}>
-                    {result}
-                </result_iteration_{iterations}>
-                """
-                )
-                resources = parse_xml(result, "resource")
-                datasources = []
-                pipes = []
-                for resource_xml in resources:
-                    resource_type = extract_xml(resource_xml, "type")
-                    name = extract_xml(resource_xml, "name")
-                    content = extract_xml(resource_xml, "content")
-                    resource = {
-                        "name": name,
-                        "content": content,
-                    }
-                    if resource_type.lower() == "datasource":
-                        datasources.append(resource)
-                    elif resource_type.lower() == "pipe":
-                        pipes.append(resource)
-
-                for ds in datasources:
-                    content = ds["content"].replace("```", "")
-                    filename = f"{ds['name']}.datasource"
-                    datasource_path = generate_datafile(
-                        content,
-                        filename=filename,
-                        data=None,
-                        _format="ndjson",
-                        force=True,
-                        folder=folder,
-                    )
-                    generated_paths.append(datasource_path)
-                for pipe in pipes:
-                    content = pipe["content"].replace("```", "")
-                    pipe_path = generate_pipe_file(pipe["name"], content, folder)
-                    generated_paths.append(pipe_path)
-                if skip:
-                    break
-                iterations += 1
-
-        if iterations == 10:
-            click.echo(FeedbackManager.info(message="Too many iterations. Change the prompt and try again."))
+        result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
+        result = extract_xml(result, "response")
+        resources = parse_xml(result, "resource")
+        datasources = []
+        pipes = []
+        for resource_xml in resources:
+            resource_type = extract_xml(resource_xml, "type")
+            name = extract_xml(resource_xml, "name")
+            content = extract_xml(resource_xml, "content")
+            resource = {
+                "name": name,
+                "content": content,
+            }
+            if resource_type.lower() == "datasource":
+                datasources.append(resource)
+            elif resource_type.lower() == "pipe":
+                pipes.append(resource)
+
+        for ds in datasources:
+            content = ds["content"].replace("```", "")
+            filename = f"{ds['name']}.datasource"
+            generate_datafile(
+                content,
+                filename=filename,
+                data=None,
+                _format="ndjson",
+                force=True,
+                folder=folder,
+            )
+
+        for pipe in pipes:
+            content = pipe["content"].replace("```", "")
+            generate_pipe_file(pipe["name"], content, folder)
 
         return result
 
@@ -313,14 +270,19 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
     def is_sink(content: str) -> bool:
         return re.search(r"TYPE sink", content, re.IGNORECASE) is not None
 
+    def is_endpoint(content: str) -> bool:
+        return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
+
     if is_copy(content):
         pathname = "copies"
     elif is_materialization(content):
         pathname = "materializations"
     elif is_sink(content):
         pathname = "sinks"
-    else:
+    elif is_endpoint(content):
         pathname = "endpoints"
+    else:
+        pathname = "pipes"
 
     base = Path(folder) / pathname
     if not base.exists():

tinybird/tb/modules/datafile/build_common.py

@@ -94,7 +94,7 @@ async def update_tags_in_resource(rs: Dict[str, Any], resource_type: str, client
             resource_name = persisted_ds.get("name", "")
         except DoesNotExistException:
             click.echo(
-                FeedbackManager.error_tag_generic("Could not get the latest Data Source info for updating its tags.")
+                FeedbackManager.error_tag_generic("Could not get the latest data source info for updating its tags.")
             )
     elif resource_type == "pipe":
         pipe_name = rs["name"]

tinybird/tb/modules/datafile/build_datasource.py

@@ -60,7 +60,7 @@ async def new_ds(
 
     if engine_param.lower() == "join":
         deprecation_notice = FeedbackManager.warning_deprecated(
-            warning="Data Sources with Join engine are deprecated and will be removed in the next major release of tinybird-cli. Use MergeTree instead."
+            warning="Data sources with Join engine are deprecated and will be removed in the next major release of tinybird-cli. Use MergeTree instead."
         )
         click.echo(deprecation_notice)
 

tinybird/tb/modules/datafile/common.py

@@ -20,6 +20,7 @@ from string import Template
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, cast
 
 import click
+from croniter import croniter
 from mypy_extensions import KwArg, VarArg
 
 from tinybird.ch_utils.engine import ENABLED_ENGINES
@@ -200,6 +201,20 @@ class Datafile:
     def set_kind(self, kind: DatafileKind):
         self.kind = kind
 
+    def validate_copy_node(self, node: Dict[str, Any]):
+        if "target_datasource" not in node:
+            raise DatafileValidationError("COPY node missing target datasource")
+        # copy mode must be append or replace
+        if node.get("mode") and node["mode"] not in ["append", "replace"]:
+            raise DatafileValidationError("COPY node mode must be append or replace")
+        # copy schedule must be @on-demand or a cron-expression
+        if (
+            node.get("copy_schedule")
+            and node["copy_schedule"] != ON_DEMAND
+            and not croniter.is_valid(node["copy_schedule"])
+        ):
+            raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
+
     def validate(self):
         if self.kind == DatafileKind.pipe:
             # TODO(eclbg):
@@ -208,7 +223,7 @@ class Datafile:
             #  [x] Materialized nodes have target datasource
             #  [x] Only one materialized node
             #  [x] Only one node of any specific type
-            #  [ ] ...
+            #  (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
             for node in self.nodes:
                 if "sql" not in node:
                     raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
@@ -220,6 +235,8 @@ class Datafile:
                         raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
                 if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
                     raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
+                if node.get("type", "").lower() == PipeNodeTypes.COPY:
+                    self.validate_copy_node(node)
         elif self.kind == DatafileKind.datasource:
             # TODO(eclbg):
             #  [x] Just one node
@@ -232,8 +249,6 @@ class Datafile:
             node = self.nodes[0]
             if "schema" not in node:
                 raise DatafileValidationError("SCHEMA is mandatory")
-            if "engine" not in node:
-                raise DatafileValidationError("ENGINE is mandatory")
         else:
             # We cannot validate a datafile whose kind is unknown
             pass

tinybird/tb/modules/datafile/pipe_checker.py

@@ -412,7 +412,7 @@ class PipeCheckerRunner:
             )
 
         result = PipeCheckerTextTestResult(
-            self.checker_stream_result_class(sys.stdout),  # type: ignore
+            self.checker_stream_result_class(sys.stdout),
             descriptions=True,
             verbosity=2,
             custom_output=custom_output,

tinybird/tb/modules/datasource.py

@@ -34,7 +34,7 @@ def datasource(ctx):
 
 
 @datasource.command(name="ls")
-@click.option("--match", default=None, help="Retrieve any resources matching the pattern. eg --match _test")
+@click.option("--match", default=None, help="Retrieve any resources matching the pattern. For example, --match _test")
 @click.option(
     "--format",
     "format_",
@@ -116,7 +116,7 @@ async def datasource_append(
     concurrency: int,
 ):
     """
-    Appends data to an existing Data Source from URL, local file  or a connector
+    Appends data to an existing data source from URL, local file  or a connector
 
     - Load from URL `tb datasource append [datasource_name] https://url_to_csv`