tinybird 0.0.1.dev34__tar.gz → 0.0.1.dev36__tar.gz

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird
-Version: 0.0.1.dev34
+Version: 0.0.1.dev36
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/context.py

@@ -7,7 +7,7 @@ if TYPE_CHECKING:
 
 workspace_id: ContextVar[str] = ContextVar("workspace_id")
 workspace: ContextVar["User"] = ContextVar("workspace")
-table_id: ContextVar[str] = ContextVar("table_id")
+datasource_id: ContextVar[str] = ContextVar("datasource_id")
 hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
 hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")
 use_gatherer: ContextVar[bool] = ContextVar("use_gatherer")

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/datafile.py

@@ -3434,14 +3434,33 @@ async def new_ds(
     DATASOURCE_VALID_SERVICES_TO_UPDATE = ["bigquery", "snowflake"]
     if datasource_exists and service and service in [*DATASOURCE_VALID_SERVICES_TO_UPDATE, *PREVIEW_CONNECTOR_SERVICES]:
         connector_required_params = {
-            "bigquery": ["service", "cron", "external_data_source"],
+            "bigquery": ["service", "cron"],
             "snowflake": ["connector", "service", "cron", "external_data_source"],
             "s3": ["connector", "service", "cron", "bucket_uri"],
             "s3_iamrole": ["connector", "service", "cron", "bucket_uri"],
             "gcs": ["connector", "service", "cron", "bucket_uri"],
         }.get(service, [])
 
+        connector_at_least_one_required_param = {
+            "bigquery": ["external_data_source", "query"],
+        }.get(service, [])
+
+        if connector_at_least_one_required_param and not any(
+            key in ds_params for key in connector_at_least_one_required_param
+        ):
+            params = [
+                (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
+                for param in connector_at_least_one_required_param
+            ]
+            click.echo(FeedbackManager.error_updating_connector_missing_at_least_one_param(param=" or ".join(params)))
+            return
+
         if not all(key in ds_params for key in connector_required_params):
+            params = [
+                (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
+                for param in connector_required_params
+            ]
+            click.echo(FeedbackManager.error_updating_connector_missing_params(param=", ".join(params)))
             return
 
         connector = ds_params.get("connector", None)
@@ -5589,3 +5608,38 @@ def is_file_a_datasource(filename: str) -> bool:
                 return True
 
     return False
+
+
+def update_connector_params(service: str, ds_params: Dict[str, Any], connector_required_params: List[str]) -> None:
+    """
+    Update connector parameters for a given service, ensuring required parameters exist.
+
+    :param service: The name of the service (e.g., 'bigquery').
+    :param ds_params: The data source parameters to be checked.
+    :param connector_required_params: The list of required parameters for the connector.
+    :return: None
+    """
+
+    connector_at_least_one_required_param: List[str] = {
+        "bigquery": ["external_data_source", "query"],
+    }.get(service, [])
+
+    # Handle the "at least one param" requirement
+    if connector_at_least_one_required_param and not any(
+        key in ds_params for key in connector_at_least_one_required_param
+    ):
+        params = [
+            (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
+            for param in connector_at_least_one_required_param
+        ]
+        click.echo(FeedbackManager.error_updating_connector_missing_at_least_one_param(param=" or ".join(params)))
+        return
+
+    # Handle the mandatory params requirement
+    if not all(key in ds_params for key in connector_required_params):
+        params = [
+            (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
+            for param in connector_required_params
+        ]
+        click.echo(FeedbackManager.error_updating_connector_missing_params(param=", ".join(params)))
+        return

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/feedback_manager.py

@@ -130,6 +130,12 @@ class FeedbackManager:
     error_remove_no_endpoint = error_message("Pipe does not have any endpoint")
     error_updating_pipe = error_message("Failed updating pipe {error}")
     error_updating_connector_not_supported = error_message("Changing {param} is not currently supported")
+    error_updating_connector_missing_at_least_one_param = error_message(
+        "Connection settings not updated. Connection info should have at least one of {param} settings"
+    )
+    error_updating_connector_missing_params = error_message(
+        "Connection settings not updated. Connection info should have {param} settings"
+    )
     error_removing_node = error_message("Failed removing node from pipe {pipe}: {error}")
     error_pushing_pipe = error_message("Failed pushing pipe {pipe}: {error}")
     error_creating_endpoint = error_message("Failed creating endpoint in node {node} on pipe {pipe}: {error}")

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/prompts.py

@@ -641,8 +641,11 @@ The previous instructions are explanations of how things work in Tinybird. Answe
 
 datasource_instructions = """
 <datasource_file_instructions>
+    - Content cannot be empty.
     - The datasource names must be unique.
     - No indentation is allowed for property names: DESCRIPTION, SCHEMA, ENGINE, ENGINE_PARTITION_KEY, ENGINE_SORTING_KEY, etc.
+    - Use MergeTree engine by default.
+    - Use AggregatingMergeTree engine when the datasource is the target of a materialized pipe.
 </datasource_file_instructions>
 """
 
@@ -699,6 +702,7 @@ sql_instructions = """
     </valid_query_with_parameters_with_%_on_top>
     - The Parameter functions like this one {{{{String(my_param_name,default_value)}}}} can be one of the following: String, DateTime, Date, Float32, Float64, Int, Integer, UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
     - Parameter names must be different from column names. Pass always the param name and a default value to the function.
+    - Use ALWAYS hardcoded values for default values for parameters.
     - Code inside the template {{{{template_expression}}}} follows the rules of Tornado templating language so no module is allowed to be imported. So for example you can't use now() as default value for a DateTime parameter. You need an if else block like this:
     <invalid_condition_with_now>
     AND timestamp BETWEEN {{DateTime(start_date, now() - interval 30 day)}} AND {{DateTime(end_date, now())}}
@@ -732,6 +736,8 @@ sql_instructions = """
     - When aliasing a column, use first the column name and then the alias.
     - General functions and aggregate functions are case sensitive.
     - Character insensitive functions are case insensitive.
+    - When you use defined function with a paremeter inside, do NOT add quotes around the parameter.
+    - Parameters are never quoted in any case.
 </sql_instructions>
 """.format(
     general_functions=general_functions,

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/sql_toolset.py

@@ -3,7 +3,7 @@ import logging
 from collections import defaultdict
 from datetime import datetime
 from functools import lru_cache
-from typing import Any, FrozenSet, List, Optional, Set, Tuple
+from typing import FrozenSet, List, Optional, Set, Tuple
 
 from chtoolset import query as chquery
 from toposort import toposort
@@ -172,7 +172,7 @@ def tables_or_sql(replacement: dict, table_functions=False) -> set:
         return {replacement}
 
 
-def _separate_as_tuple_if_contains_database_and_table(definition: str) -> Any:
+def _separate_as_tuple_if_contains_database_and_table(definition: str) -> str | Tuple[str, str]:
     if "." in definition:
         database_and_table_separated = definition.split(".")
         return database_and_table_separated[0], database_and_table_separated[1]
@@ -219,6 +219,7 @@ def replace_tables(
     output_one_line: bool = False,
     timestamp: Optional[datetime] = None,
     function_allow_list: Optional[FrozenSet[str]] = None,
+    original_replacements: Optional[dict] = None,
 ) -> str:
     """
     Given a query and a list of table replacements, returns the query after applying the table replacements.
@@ -239,6 +240,12 @@ def replace_tables(
         _replacements[rk] = r if isinstance(r, tuple) else (default_database, r)
         _replaced_with.add(r)
 
+    if original_replacements:
+        # Some replacements have been expanded by filters and turned to a query str, but we need to send the original
+        # ones to is_invalid_resource()
+        for r in original_replacements.values():
+            _replaced_with.add(r)
+
     deps: defaultdict = defaultdict(set)
     _tables = sql_get_used_tables(
         sql,

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '0.0.1.dev34'
-__revision__ = '08098c9'
+__version__ = '0.0.1.dev36'
+__revision__ = '5ad3ab4'

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/cli.py

@@ -9,10 +9,10 @@ import tinybird.tb.modules.build
 import tinybird.tb.modules.build_client
 import tinybird.tb.modules.cli
 import tinybird.tb.modules.common
-import tinybird.tb.modules.connection
 import tinybird.tb.modules.create
 import tinybird.tb.modules.datasource
-import tinybird.tb.modules.deploy
+import tinybird.tb.modules.deployment
+import tinybird.tb.modules.endpoint
 import tinybird.tb.modules.fmt
 import tinybird.tb.modules.job
 import tinybird.tb.modules.local

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/modules/build.py

@@ -2,7 +2,9 @@ import asyncio
 import glob
 import json
 import logging
+import os
 import threading
+import time
 from pathlib import Path
 from typing import List
 
@@ -12,6 +14,7 @@ import requests
 from tinybird.client import TinyB
 from tinybird.tb.modules.cli import cli
 from tinybird.tb.modules.common import push_data
+from tinybird.tb.modules.datafile.fixture import build_fixture_name, get_fixture_dir
 from tinybird.tb.modules.feedback_manager import FeedbackManager
 from tinybird.tb.modules.local_common import get_tinybird_local_client
 from tinybird.tb.modules.shell import Shell
@@ -19,7 +22,7 @@ from tinybird.tb.modules.watch import watch_project
 
 
 @cli.command()
-@click.option("--folder", type=str, default=".")
+@click.option("--folder", type=str, default=os.getcwd())
 @click.option("--watch", is_flag=True, default=False, help="Watch for changes and rebuild automatically")
 def build(folder: str, watch: bool) -> None:
     """
@@ -27,12 +30,19 @@ def build(folder: str, watch: bool) -> None:
     """
 
     tb_client = asyncio.run(get_tinybird_local_client(folder))
+    click.echo(FeedbackManager.highlight(message="\n» Building project..."))
+
+    time_start = time.time()
 
     def process() -> None:
         build_project(folder, tb_client)
 
     process()
 
+    time_end = time.time()
+    elapsed_time = time_end - time_start
+    click.echo(FeedbackManager.success(message=f"\n✓ Build completed in {elapsed_time:.1f}s"))
+
     if watch:
         shell = Shell(folder=folder, client=tb_client)
         click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
@@ -71,6 +81,7 @@ def build_project(folder: str, tb_client: TinyB) -> None:
         fds = []
         project_path = Path(folder)
         project_files = get_project_files(project_path)
+
         for file_path in project_files:
             relative_path = str(Path(file_path).relative_to(project_path))
             fd = open(file_path, "rb")
@@ -91,7 +102,34 @@ def build_project(folder: str, tb_client: TinyB) -> None:
 
         build_result = result.get("result")
         if build_result == "success":
-            click.echo(FeedbackManager.success(message="Build completed successfully"))
+            datasources = result.get("datasources", [])
+            pipes = result.get("pipes", [])
+            for ds in datasources:
+                ds_path = next((p for p in project_files if p.endswith(ds.get("name") + ".datasource")), None)
+                if ds_path:
+                    ds_path = ds_path.replace(f"{folder}/", "")
+                    click.echo(FeedbackManager.info(message=f"✓ {ds_path} created"))
+            for pipe in pipes:
+                pipe_name = pipe.get("name")
+                pipe_path = next((p for p in project_files if p.endswith(pipe_name + ".pipe")), None)
+                if pipe_path:
+                    pipe_path = pipe_path.replace(f"{folder}/", "")
+                    click.echo(FeedbackManager.info(message=f"✓ {pipe_path} created"))
+
+            for filename in project_files:
+                if filename.endswith(".datasource"):
+                    ds_path = Path(filename)
+                    ds_name = ds_path.stem
+                    name = build_fixture_name(filename, ds_name, ds_path.read_text())
+                    fixture_folder = get_fixture_dir(folder)
+                    fixture_path = fixture_folder / f"{name}.ndjson"
+
+                    if not fixture_path.exists():
+                        fixture_path = fixture_folder / f"{ds_name}.ndjson"
+
+                    if fixture_path.exists():
+                        append_fixture(tb_client, ds_name, str(fixture_path))
+
         elif build_result == "failed":
             click.echo(FeedbackManager.error(message="Build failed"))
             build_errors = result.get("errors")
@@ -101,24 +139,27 @@ def build_project(folder: str, tb_client: TinyB) -> None:
                 click.echo(FeedbackManager.error(message=error_msg))
         else:
             click.echo(FeedbackManager.error(message=f"Unknown build result. Error: {result.get('error')}"))
+
     except Exception as e:
-        click.echo(FeedbackManager.error_exception(error="Error building project: " + str(e)))
+        click.echo(FeedbackManager.error_exception(error="Error: " + str(e)))
     finally:
         for fd in fds:
             fd.close()
 
 
-async def append_fixture(
+def append_fixture(
     tb_client: TinyB,
     datasource_name: str,
     url: str,
 ):
-    await tb_client.datasource_truncate(datasource_name)
-    await push_data(
-        tb_client,
-        datasource_name,
-        url,
-        mode="append",
-        concurrency=1,
-        silent=True,
+    asyncio.run(tb_client.datasource_truncate(datasource_name))
+    asyncio.run(
+        push_data(
+            tb_client,
+            datasource_name,
+            url,
+            mode="append",
+            concurrency=1,
+            silent=True,
+        )
     )

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/modules/cli.py

@@ -67,58 +67,13 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
 )
 @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
-@click.option("--gcp-project-id", help="The Google Cloud project ID", hidden=True)
-@click.option(
-    "--gcs-bucket", help="The Google Cloud Storage bucket to write temp files when using the connectors", hidden=True
-)
-@click.option(
-    "--google-application-credentials",
-    envvar="GOOGLE_APPLICATION_CREDENTIALS",
-    help="Set GOOGLE_APPLICATION_CREDENTIALS",
-    hidden=True,
-)
-@click.option("--sf-account", help="The Snowflake Account (e.g. your-domain.west-europe.azure)", hidden=True)
-@click.option("--sf-warehouse", help="The Snowflake warehouse name", hidden=True)
-@click.option("--sf-database", help="The Snowflake database name", hidden=True)
-@click.option("--sf-schema", help="The Snowflake schema name", hidden=True)
-@click.option("--sf-role", help="The Snowflake role name", hidden=True)
-@click.option("--sf-user", help="The Snowflake user name", hidden=True)
-@click.option("--sf-password", help="The Snowflake password", hidden=True)
-@click.option(
-    "--sf-storage-integration",
-    help="The Snowflake GCS storage integration name (leave empty to auto-generate one)",
-    hidden=True,
-)
-@click.option("--sf-stage", help="The Snowflake GCS stage name (leave empty to auto-generate one)", hidden=True)
-@click.option(
-    "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
-)
 @click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
-@click.option("--prod/--local", is_flag=True, default=False, help="Run against production or local")
+@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens")
+@click.option("--prod", is_flag=True, default=False, help="Run against production")
 @click.version_option(version=VERSION)
 @click.pass_context
 @coro
-async def cli(
-    ctx: Context,
-    debug: bool,
-    token: str,
-    host: str,
-    gcp_project_id: str,
-    gcs_bucket: str,
-    google_application_credentials: str,
-    sf_account: str,
-    sf_warehouse: str,
-    sf_database: str,
-    sf_schema: str,
-    sf_role: str,
-    sf_user: str,
-    sf_password: str,
-    sf_storage_integration: str,
-    sf_stage,
-    with_headers: bool,
-    show_tokens: bool,
-    prod: bool,
-) -> None:
+async def cli(ctx: Context, debug: bool, token: str, host: str, show_tokens: bool, prod: bool) -> None:
     """
     Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output.
     """
@@ -166,51 +121,9 @@ async def cli(
 
     ctx.ensure_object(dict)["config"] = config
 
-    if ctx.invoked_subcommand == "auth":
-        return
-
-    from tinybird.connectors import create_connector
-
-    if gcp_project_id and gcs_bucket and google_application_credentials and not sf_account:
-        bq_config = {
-            "project_id": gcp_project_id,
-            "bucket_name": gcs_bucket,
-            "service_account": google_application_credentials,
-            "with_headers": with_headers,
-        }
-        ctx.ensure_object(dict)["bigquery"] = create_connector("bigquery", bq_config)
-    if (
-        sf_account
-        and sf_warehouse
-        and sf_database
-        and sf_schema
-        and sf_role
-        and sf_user
-        and sf_password
-        and gcs_bucket
-        and google_application_credentials
-        and gcp_project_id
-    ):
-        sf_config = {
-            "account": sf_account,
-            "warehouse": sf_warehouse,
-            "database": sf_database,
-            "schema": sf_schema,
-            "role": sf_role,
-            "user": sf_user,
-            "password": sf_password,
-            "storage_integration": sf_storage_integration,
-            "stage": sf_stage,
-            "bucket_name": gcs_bucket,
-            "service_account": google_application_credentials,
-            "project_id": gcp_project_id,
-            "with_headers": with_headers,
-        }
-        ctx.ensure_object(dict)["snowflake"] = create_connector("snowflake", sf_config)
-
     logging.debug("debug enabled")
 
-    skip_client = ctx.invoked_subcommand in ["login", "workspace", "local"]
+    skip_client = ctx.invoked_subcommand in ["auth", "login", "workspace", "local", "build"]
     client = await create_ctx_client(config, prod, skip_client)
 
     if client:
@@ -487,7 +400,7 @@ async def sql(
 
 @cli.command(hidden=True)
 @click.argument("prompt")
-@click.option("--folder", default=".", help="The folder to use for the project")
+@click.option("--folder", default=os.getcwd(), help="The folder to use for the project")
 @coro
 async def ask(prompt: str, folder: str) -> None:
     """Ask things about your data project."""
@@ -529,8 +442,8 @@ async def ask(prompt: str, folder: str) -> None:
         )
 
         client = config.get_client()
-        llm = LLM(user_token=user_token, client=client)
-        click.echo(await llm.ask(system_prompt=ask_prompt(resources_xml), prompt=prompt))
+        llm = LLM(user_token=user_token, host=client.host)
+        click.echo(llm.ask(system_prompt=ask_prompt(resources_xml), prompt=prompt))
     except Exception as e:
         raise CLIException(FeedbackManager.error_exception(error=e))
 

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/modules/create.py

@@ -103,13 +103,13 @@ async def create(
                 datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
                 for datasource_file in datasource_files:
                     datasource_path = Path(folder) / "datasources" / datasource_file
-                    llm = LLM(user_token=user_token, client=tb_client)
+                    llm = LLM(user_token=user_token, host=tb_client.host)
                     datasource_name = datasource_path.stem
                     datasource_content = datasource_path.read_text()
                     has_json_path = "`json:" in datasource_content
                     if has_json_path:
                         prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
-                        response = await llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
+                        response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
                         sql = extract_xml(response, "sql")
                         sql = sql.split("FORMAT")[0]
                         result = await local_client.query(f"{sql} FORMAT JSON")
@@ -205,8 +205,8 @@ TYPE ENDPOINT
                 ]
             ]
         )
-        llm = LLM(user_token=user_token, client=tb_client)
-        result = await llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
+        llm = LLM(user_token=user_token, host=tb_client.host)
+        result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
         result = extract_xml(result, "response")
         resources = parse_xml(result, "resource")
         datasources = []

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/modules/datafile/build.py

@@ -44,7 +44,7 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
 from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
 from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
 from tinybird.tb.modules.feedback_manager import FeedbackManager
-from tinybird.tb.modules.local_common import get_client_config_for_build
+from tinybird.tb.modules.local_common import get_tinybird_local_config
 
 
 async def folder_build(
@@ -58,7 +58,7 @@ async def folder_build(
     local_ws: Optional[Dict[str, Any]] = None,
     watch: bool = False,
 ):
-    config = await get_client_config_for_build(folder)
+    config = await get_tinybird_local_config(folder)
     build = True
     dry_run = False
     force = True

{tinybird-0.0.1.dev34 → tinybird-0.0.1.dev36}/tinybird/tb/modules/datafile/common.py

@@ -204,6 +204,10 @@ class Datafile:
         if self.kind == DatafileKind.pipe:
             # TODO(eclbg):
             #  [x] node names are unique
+            #  [x] SQL in all nodes
+            #  [x] Materialized nodes have target datasource
+            #  [x] Only one materialized node
+            #  [ ] Only one node of any specific type
             #  [ ] ...
             repeated_node_names = [
                 name for name, count in filter(lambda x: x[1] > 1, Counter(n["name"] for n in self.nodes).items())
@@ -212,7 +216,19 @@ class Datafile:
                 raise DatafileValidationError(
                     f"Pipe node names must be unique. These names are repeated: {repeated_node_names}"
                 )
-            pass
+            for node in self.nodes:
+                if "sql" not in node:
+                    raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
+            materialized_nodes_count = 0
+            for node in self.nodes:
+                if node.get("type", "").lower() == "materialized":
+                    materialized_nodes_count += 1
+                    if materialized_nodes_count > 1:
+                        raise DatafileValidationError("Multiple materialized nodes in pipe. There can only be one")
+                    if "datasource" not in node:
+                        raise DatafileValidationError(
+                            f"Materialized node {repr(node['name'])} missing target datasource"
+                        )
         elif self.kind == DatafileKind.datasource:
             # TODO(eclbg):
             #  [x] Just one node