tinybird-cli 6.5.4.dev0__tar.gz → 6.5.5.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/PKG-INFO +10 -1
  2. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/__cli__.py +2 -2
  3. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/ch_utils/engine.py +6 -25
  4. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/client.py +34 -10
  5. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/config.py +0 -6
  6. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/datafile_common.py +50 -59
  7. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/datatypes.py +0 -4
  8. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/feedback_manager.py +3 -0
  9. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql.py +0 -24
  10. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_template.py +92 -45
  11. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_toolset.py +26 -7
  12. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/branch.py +0 -1
  13. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/common.py +15 -17
  14. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/config.py +1 -15
  15. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/datasource.py +68 -4
  16. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -9
  17. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace.py +0 -1
  18. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tornado_template.py +5 -5
  19. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/PKG-INFO +10 -1
  20. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/setup.cfg +0 -0
  21. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/ch_utils/constants.py +0 -0
  22. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/check_pypi.py +0 -0
  23. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/context.py +0 -0
  24. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/git_settings.py +0 -0
  25. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/sql_template_fmt.py +0 -0
  26. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/syncasync.py +0 -0
  27. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli.py +0 -0
  28. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/auth.py +0 -0
  29. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/cicd.py +0 -0
  30. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/cli.py +0 -0
  31. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/connection.py +0 -0
  32. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/exceptions.py +0 -0
  33. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/fmt.py +0 -0
  34. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/job.py +0 -0
  35. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/pipe.py +0 -0
  36. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/regions.py +0 -0
  37. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/tag.py +0 -0
  38. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/telemetry.py +0 -0
  39. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/test.py +0 -0
  40. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/token.py +0 -0
  41. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  42. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/SOURCES.txt +0 -0
  43. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/dependency_links.txt +0 -0
  44. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/entry_points.txt +0 -0
  45. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/requires.txt +0 -0
  46. {tinybird_cli-6.5.4.dev0 → tinybird_cli-6.5.5.dev0}/tinybird_cli.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird_cli
3
- Version: 6.5.4.dev0
3
+ Version: 6.5.5.dev0
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli
6
6
  Author: Tinybird
@@ -43,10 +43,19 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
43
43
  Changelog
44
44
  ----------
45
45
 
46
+ 6.5.4
47
+ ***********
48
+
49
+ - `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
50
+ - `Fixed` Internal improvements to secret template rendering used during datafile parsing.
51
+ - `Fixed` `tb push` now rejects pipes that declare a parameter without a valid name (e.g. a numeric-literal cast like ``{{Float32(1682892000.0)}}``), instead of creating an unusable parameter.
52
+
46
53
  6.5.3
47
54
  ***********
48
55
 
56
+ - `Removed` `bi_stats_rt` and `bi_stats` from service datasource catalogs and dependency ignore lists.
49
57
  - `Added` ``tb datasource delete`` now accepts ``--lightweight-delete``, ``--partition`` and ``--projection-mode``, backed by the ``POST /v1/datasources/{name}/delete`` endpoint that uses ClickHouse's lightweight ``DELETE FROM`` instead of the heavy ``ALTER TABLE ... DELETE`` mutation. Lightweight delete is synchronous by default and returns the affected row count plus a partition-level mutation block; pass ``--no-wait`` to enqueue a job and receive the v0-compatible envelope.
58
+ - `Changed` Upgraded Tornado to 6.5.5.
50
59
 
51
60
  6.5.1
52
61
  ***********
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '6.5.4.dev0'
8
- __revision__ = 'a87d7b3'
7
+ __version__ = '6.5.5.dev0'
8
+ __revision__ = '5d2bbc5'
@@ -121,6 +121,12 @@ class TableDetails:
121
121
  def is_mergetree_family(self) -> bool:
122
122
  return self.engine is not None and "mergetree" in self.engine.lower()
123
123
 
124
+ def is_view(self) -> bool:
125
+ return self.original_engine == "View"
126
+
127
+ def is_merge_final_engine(self) -> bool:
128
+ return self.original_engine == "MergeFinal"
129
+
124
130
  def supports_alter_add_column(self) -> bool:
125
131
  return self.is_mergetree_family() or (self.engine is not None and self.engine.lower() == "null")
126
132
 
@@ -354,31 +360,6 @@ def sorting_key_is_valid(columns: List[Dict[str, Any]], value: Optional[str]) ->
354
360
  return value
355
361
 
356
362
 
357
- def case_insensitive_check(valid_values: List[str]) -> Callable[[List[Dict[str, Any]], str], Optional[str]]:
358
- """
359
- >>> valid_values = ['ANY', 'ALL']
360
- >>> checker = case_insensitive_check(valid_values)
361
- >>> checker([],'ALL')
362
-
363
- >>> valid_values = ['ANY', 'ALL']
364
- >>> checker = case_insensitive_check(valid_values)
365
- >>> checker([],'any')
366
-
367
- >>> valid_values = ['ANY', 'ALL']
368
- >>> checker = case_insensitive_check(valid_values)
369
- >>> checker([],'foo')
370
- Traceback (most recent call last):
371
- ...
372
- ValueError: valid values are ANY, ALL
373
- """
374
-
375
- def checker(columns: List[Dict[str, Any]], value: str):
376
- if value.upper() not in valid_values:
377
- raise ValueError(f"valid values are {', '.join(valid_values)}")
378
-
379
- return checker
380
-
381
-
382
363
  # [PARTITION BY expr]
383
364
  # [ORDER BY expr]
384
365
  # [PRIMARY KEY expr]
@@ -441,6 +441,7 @@ class TinyB:
441
441
  sql_condition: Optional[str] = None,
442
442
  format: str = "csv",
443
443
  replace_options: Optional[Set[str]] = None,
444
+ use_v1: bool = False,
444
445
  ):
445
446
  params = {"name": datasource_name, "mode": mode, "format": format, "debug": "blocks_block_log"}
446
447
 
@@ -452,20 +453,43 @@ class TinyB:
452
453
 
453
454
  async with aiofiles.open(file, "rb") as content:
454
455
  file_content = await content.read()
455
- if format == "csv":
456
- files = {"csv": ("csv", file_content)}
457
- else:
458
- files = {"ndjson": ("ndjson", file_content)}
459
456
 
460
- res = await self._req(
461
- f"v0/datasources?{urlencode(params, safe='')}",
462
- files=files,
457
+ if use_v1:
458
+ v1_params = {"format": format}
459
+ if sql_condition:
460
+ v1_params["replace_condition"] = sql_condition
461
+ if replace_options:
462
+ for option in list(replace_options):
463
+ v1_params[option] = "true"
464
+ content_types = {
465
+ "csv": "text/csv",
466
+ "ndjson": "application/x-ndjson",
467
+ "parquet": "application/vnd.apache.parquet",
468
+ }
469
+ headers = {"Content-Type": content_types[format]}
470
+ if str(file).endswith(".gz"):
471
+ headers["Content-Encoding"] = "gzip"
472
+ return await self._req(
473
+ f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(v1_params, safe='')}",
474
+ data=file_content,
475
+ headers=headers,
463
476
  method="POST",
464
477
  )
465
- if status_callback:
466
- status_callback(res)
467
478
 
468
- return res
479
+ if format == "csv":
480
+ files = {"csv": ("csv", file_content)}
481
+ else:
482
+ files = {"ndjson": ("ndjson", file_content)}
483
+
484
+ res = await self._req(
485
+ f"v0/datasources?{urlencode(params, safe='')}",
486
+ files=files,
487
+ method="POST",
488
+ )
489
+ if status_callback:
490
+ status_callback(res)
491
+
492
+ return res
469
493
 
470
494
  async def datasource_truncate(self, datasource_name: str):
471
495
  return await self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
@@ -112,12 +112,6 @@ async def get_config(
112
112
  return config
113
113
 
114
114
 
115
- async def write_config(config: Dict[str, Any], dest_file: str = ".tinyb"):
116
- config_file = Path(getcwd()) / dest_file
117
- async with aiofiles.open(config_file, "w") as file:
118
- await file.write(json.dumps(config, indent=4, sort_keys=True))
119
-
120
-
121
115
  def get_display_host(ui_host: str):
122
116
  return LEGACY_HOSTS.get(ui_host, ui_host)
123
117
 
@@ -9,7 +9,7 @@ import aiofiles
9
9
  from requests import Response
10
10
 
11
11
  try:
12
- from colorama import Back, Fore, Style, init
12
+ from colorama import Fore, init
13
13
 
14
14
  init()
15
15
  except ImportError: # fallback so that the imported classes always exist
@@ -18,7 +18,7 @@ except ImportError: # fallback so that the imported classes always exist
18
18
  def __getattr__(self, name):
19
19
  return ""
20
20
 
21
- Fore = Back = Style = ColorFallback()
21
+ Fore = ColorFallback()
22
22
 
23
23
  import difflib
24
24
  import glob
@@ -94,8 +94,6 @@ INTERNAL_TABLES: Tuple[str, ...] = (
94
94
  "kafka_ops_log",
95
95
  "datasources_storage",
96
96
  "endpoint_errors",
97
- "bi_stats_rt",
98
- "bi_stats",
99
97
  )
100
98
 
101
99
 
@@ -306,10 +304,10 @@ class ValidationException(Exception):
306
304
  super().__init__(err)
307
305
 
308
306
 
309
- def sizeof_fmt(num: Union[int, float], suffix: str = "b") -> str:
307
+ def sizeof_fmt(num: float, suffix: str = "b") -> str:
310
308
  """Readable file size
311
309
  :param num: Bytes value
312
- :type num: int
310
+ :type num: float
313
311
  :param suffix: Unit suffix (optionnal) default = o
314
312
  :type suffix: str
315
313
  :rtype: str
@@ -854,9 +852,24 @@ def parse_pipe(
854
852
  doc = parse(s, basepath=basepath, replace_includes=replace_includes, skip_eval=skip_eval)
855
853
  for node in doc.nodes:
856
854
  sql = node.get("sql", "")
857
- if sql.strip()[0] == "%":
858
- sql, _, variable_warnings = render_sql_template(sql[1:], test_mode=True, name=node["name"])
855
+ source_sql = sql.strip()
856
+ if not source_sql:
857
+ raise click.ClickException(
858
+ FeedbackManager.error_parsing_node(node=node["name"], pipe=filename, error="Empty query")
859
+ )
860
+ if source_sql.startswith("%"):
861
+ sql, _, variable_warnings = render_sql_template(source_sql[1:], test_mode=True, name=node["name"])
859
862
  doc.warnings = variable_warnings
863
+ if not sql.strip():
864
+ raise click.ClickException(
865
+ FeedbackManager.error_parsing_node(
866
+ node=node["name"],
867
+ pipe=filename,
868
+ error="Template renders to an empty query without parameters",
869
+ )
870
+ )
871
+ else:
872
+ sql = source_sql
860
873
  # it'll fail with a ModuleNotFoundError when the toolset is not available but it returns the parsed doc
861
874
  from tinybird.sql_toolset import format_sql as toolset_format_sql
862
875
 
@@ -868,13 +881,15 @@ def parse_pipe(
868
881
  )
869
882
  )
870
883
  except ValueError as e:
871
- t, template_variables, _ = get_template_and_variables(sql, name=node["name"])
884
+ source_sql = node.get("sql", "")
885
+ source_sql_stripped = source_sql.strip()
886
+ t, template_variables, _ = get_template_and_variables(source_sql, name=node["name"])
872
887
 
873
- if sql.strip()[0] != "%" and len(template_variables) > 0:
888
+ if not source_sql_stripped.startswith("%") and len(template_variables) > 0:
874
889
  raise click.ClickException(FeedbackManager.error_template_start(filename=filename))
875
890
  raise click.ClickException(
876
891
  FeedbackManager.error_parsing_file(
877
- filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {sql}"
892
+ filename=filename, lineno="", error=f"{str(e)} + SQL(value error): {source_sql}"
878
893
  )
879
894
  )
880
895
  except UnClosedIfError as e:
@@ -1028,10 +1043,13 @@ def parse(
1028
1043
 
1029
1044
  parser_state.current_node["indexes"] = indexes
1030
1045
 
1031
- def assign_var(v: str) -> Callable[[VarArg(str), KwArg(Any)], None]:
1046
+ def assign_var(v: str, lowercase: bool = False) -> Callable[[VarArg(str), KwArg(Any)], None]:
1032
1047
  def _f(*args: str, **kwargs: Any):
1033
1048
  s = _unquote((" ".join(args)).strip())
1034
- parser_state.current_node[v.lower()] = eval_var(s, skip=skip_eval)
1049
+ val = eval_var(s, skip=skip_eval)
1050
+ if lowercase:
1051
+ val = val.lower()
1052
+ parser_state.current_node[v.lower()] = val
1035
1053
 
1036
1054
  return _f
1037
1055
 
@@ -1246,7 +1264,9 @@ def parse(
1246
1264
  "import_external_datasource": assign_var("import_external_datasource"),
1247
1265
  "import_bucket_uri": assign_var("import_bucket_uri"),
1248
1266
  "import_from_timestamp": assign_var("import_from_timestamp"),
1249
- "import_format": assign_var("import_format"),
1267
+ # Lowercase: format checks and executor routing downstream expect
1268
+ # lowercase values (ANALYTICS-A8M)
1269
+ "import_format": assign_var("import_format", lowercase=True),
1250
1270
  "import_query": assign_var("import_query"),
1251
1271
  "import_table_arn": assign_var("import_table_arn"),
1252
1272
  "import_export_bucket": assign_var("import_export_bucket"),
@@ -1750,7 +1770,7 @@ async def process_file(
1750
1770
  "filename": filename,
1751
1771
  "name": name + version,
1752
1772
  "nodes": nodes,
1753
- "deps": [x for x in set(deps)],
1773
+ "deps": list(set(deps)),
1754
1774
  "tokens": doc.tokens,
1755
1775
  "description": description,
1756
1776
  "warnings": doc.warnings,
@@ -2082,12 +2102,12 @@ class PipeChecker(unittest.TestCase):
2082
2102
 
2083
2103
  if self.ignore_order:
2084
2104
  current_data = (
2085
- sorted(normalize_array(current_data), key=itemgetter(*[k for k in current_data[0].keys()]))
2105
+ sorted(normalize_array(current_data), key=itemgetter(*list(current_data[0].keys())))
2086
2106
  if len(current_data) > 0
2087
2107
  else current_data
2088
2108
  )
2089
2109
  checker_data = (
2090
- sorted(normalize_array(checker_data), key=itemgetter(*[k for k in checker_data[0].keys()]))
2110
+ sorted(normalize_array(checker_data), key=itemgetter(*list(checker_data[0].keys())))
2091
2111
  if len(checker_data) > 0
2092
2112
  else checker_data
2093
2113
  )
@@ -2202,6 +2222,8 @@ class PipeCheckerRunner:
2202
2222
  FROM {pipe_stats_rt}
2203
2223
  WHERE
2204
2224
  pipe_name = '{self.pipe_name}'
2225
+ -- pipe_stats_rt retains 3 months; recent requests are enough for checks
2226
+ AND start_datetime >= now() - INTERVAL 7 DAY
2205
2227
  AND url IS NOT NULL
2206
2228
  AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
2207
2229
  AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
@@ -2229,6 +2251,8 @@ class PipeCheckerRunner:
2229
2251
  FROM {pipe_stats_rt}
2230
2252
  WHERE
2231
2253
  pipe_name = '{self.pipe_name}'
2254
+ -- pipe_stats_rt retains 3 months; recent requests are enough for checks
2255
+ AND start_datetime >= now() - INTERVAL 7 DAY
2232
2256
  AND url IS NOT NULL
2233
2257
  AND extractURLParameter(assumeNotNull(url), 'from') <> 'ui'
2234
2258
  AND extractURLParameter(assumeNotNull(url), 'pipe_checker') <> 'true'
@@ -4141,7 +4165,7 @@ async def build_graph(
4141
4165
  elif len(warnings) > 1:
4142
4166
  click.echo(
4143
4167
  FeedbackManager.warning_pipe_restricted_params(
4144
- words=", ".join(["'{}'".format(param) for param in warnings[:-1]]),
4168
+ words=", ".join([f"'{param}'" for param in warnings[:-1]]),
4145
4169
  last_word=warnings[-1],
4146
4170
  )
4147
4171
  )
@@ -4328,7 +4352,7 @@ async def folder_push(
4328
4352
 
4329
4353
  workspace_lib_paths = list(workspace_lib_paths)
4330
4354
  # include vendor libs without overriding user ones
4331
- existing_workspaces = set(x[1] for x in workspace_lib_paths)
4355
+ existing_workspaces = {x[1] for x in workspace_lib_paths}
4332
4356
  vendor_path = Path("vendor")
4333
4357
  if vendor_path.exists():
4334
4358
  for x in vendor_path.iterdir():
@@ -4566,7 +4590,7 @@ async def folder_push(
4566
4590
 
4567
4591
  if not fork_downstream:
4568
4592
  # First, we will deploy the all the resources following the dependency graph except for the endpoints
4569
- groups = [group for group in toposort(dependencies_graph)]
4593
+ groups = list(toposort(dependencies_graph))
4570
4594
  for group in groups:
4571
4595
  for name in group:
4572
4596
  if name in processed:
@@ -4592,7 +4616,7 @@ async def folder_push(
4592
4616
  processed.add(name)
4593
4617
 
4594
4618
  # Then, we will deploy the endpoints that are on the dependency graph
4595
- groups = [group for group in toposort(endpoints_dep_map)]
4619
+ groups = list(toposort(endpoints_dep_map))
4596
4620
  for group in groups:
4597
4621
  for name in group:
4598
4622
  if name not in processed:
@@ -4621,7 +4645,7 @@ async def folder_push(
4621
4645
 
4622
4646
  # First, we will deploy the datasources that need to be deployed.
4623
4647
  # We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
4624
- groups = [group for group in toposort(dependencies_graph_fork_downstream)]
4648
+ groups = list(toposort(dependencies_graph_fork_downstream))
4625
4649
  groups.reverse()
4626
4650
  for group in groups:
4627
4651
  for name in group:
@@ -4675,7 +4699,7 @@ async def folder_push(
4675
4699
  # If we have ENDPOINT_A ----> MV_PIPE_B -----> DATASOURCE_B ------> ENDPOINT_C
4676
4700
  # Where endpoint A is being used in the MV_PIPE_B, if we only modify the endpoint A
4677
4701
  # The dependencies graph will only contain the endpoint A and the MV_PIPE_B, but not the DATASOURCE_B and the ENDPOINT_C
4678
- groups = [group for group in toposort(dependencies_graph_fork_downstream)]
4702
+ groups = list(toposort(dependencies_graph_fork_downstream))
4679
4703
  for group in groups:
4680
4704
  for name in group:
4681
4705
  if name in processed or not is_endpoint(resources_to_run_fork_downstream[name]):
@@ -4684,7 +4708,7 @@ async def folder_push(
4684
4708
  endpoints_dep_map[name] = dependencies_graph_fork_downstream[name]
4685
4709
 
4686
4710
  # Now that we have the dependencies of the endpoints, we need to check that the resources has not been deployed yet and only care about the endpoints that depend on endpoints
4687
- groups = [group for group in toposort(endpoints_dep_map)]
4711
+ groups = list(toposort(endpoints_dep_map))
4688
4712
 
4689
4713
  # As we have used the forkdownstream graph to get the dependencies of the endpoints, we have all the dependencies of the endpoints
4690
4714
  # But we need to deploy the endpoints and the dependencies of the endpoints from left to right
@@ -4709,7 +4733,7 @@ async def folder_push(
4709
4733
  # Now we should have the endpoints and datasources deployed, we can deploy the rest of the pipes (copy & sinks)
4710
4734
  # We need to rely on the forkdownstream graph as it contains all the modified pipes as well as the dependencies of the pipes
4711
4735
  # In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
4712
- groups = [group for group in toposort(dependencies_graph_fork_downstream)]
4736
+ groups = list(toposort(dependencies_graph_fork_downstream))
4713
4737
  for group in groups:
4714
4738
  for name in group:
4715
4739
  if name in processed or is_materialized(resources_to_run_fork_downstream.get(name)):
@@ -4729,7 +4753,7 @@ async def folder_push(
4729
4753
  # Finally, we need to deploy the materialized views from right to left.
4730
4754
  # We need to rely on the forkdownstream graph as it contains all the modified materialized views as well as the dependencies of the materialized views
4731
4755
  # In this case, we don't need to generate a new graph as we did for the endpoints as the pipes are not going to be used as dependencies and the datasources are already deployed
4732
- groups = [group for group in toposort(dependencies_graph_fork_downstream)]
4756
+ groups = list(toposort(dependencies_graph_fork_downstream))
4733
4757
  for group in groups:
4734
4758
  for name in group:
4735
4759
  if name in processed or not is_materialized(resources_to_run_fork_downstream.get(name)):
@@ -5732,36 +5756,3 @@ def is_file_a_datasource(filename: str) -> bool:
5732
5756
  return True
5733
5757
 
5734
5758
  return False
5735
-
5736
-
5737
- def update_connector_params(service: str, ds_params: Dict[str, Any], connector_required_params: List[str]) -> None:
5738
- """
5739
- Update connector parameters for a given service, ensuring required parameters exist.
5740
-
5741
- :param service: The name of the service (e.g., 's3').
5742
- :param ds_params: The data source parameters to be checked.
5743
- :param connector_required_params: The list of required parameters for the connector.
5744
- :return: None
5745
- """
5746
-
5747
- connector_at_least_one_required_param: List[str] = []
5748
-
5749
- # Handle the "at least one param" requirement
5750
- if connector_at_least_one_required_param and not any(
5751
- key in ds_params for key in connector_at_least_one_required_param
5752
- ):
5753
- params = [
5754
- (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
5755
- for param in connector_at_least_one_required_param
5756
- ]
5757
- click.echo(FeedbackManager.error_updating_connector_missing_at_least_one_param(param=" or ".join(params)))
5758
- return
5759
-
5760
- # Handle the mandatory params requirement
5761
- if not all(key in ds_params for key in connector_required_params):
5762
- params = [
5763
- (ImportReplacements.get_datafile_param_for_linker_param(service, param) or param).upper()
5764
- for param in connector_required_params
5765
- ]
5766
- click.echo(FeedbackManager.error_updating_connector_missing_params(param=", ".join(params)))
5767
- return
@@ -108,10 +108,6 @@ def is_type_date(type_to_check: str) -> bool:
108
108
  return date_type_pattern.match(type_to_check) is not None
109
109
 
110
110
 
111
- def string_test(x: str) -> bool:
112
- return True
113
-
114
-
115
111
  def date_test(x: str) -> bool:
116
112
  return date_pattern.match(x) is not None
117
113
 
@@ -864,6 +864,9 @@ class FeedbackManager:
864
864
  success_total_rows = success_message("** Total rows in {datasource}: {total_rows}")
865
865
  success_appended_datasource = success_message("** Data appended to Data Source '{datasource}' successfully!")
866
866
  success_replaced_datasource = success_message("** Data replaced in Data Source '{datasource}' successfully!")
867
+ success_import_job_queued = success_message("** {operation} import job queued: {job_id}")
868
+ success_import_job_completed = success_message("** {operation} import completed: {job_id}")
869
+ info_import_job_status = info_message("Check status: tb job details {job_id}")
867
870
  success_auth = success_message(
868
871
  "** Auth successful! \n** Configuration written to .tinyb file, consider adding it to .gitignore"
869
872
  )
@@ -911,30 +911,6 @@ def engine_replicated_to_local(engine: str) -> str:
911
911
  return _RE_REPLICATED_MT.sub(_replace, engine.strip())
912
912
 
913
913
 
914
- def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:
915
- """
916
- >>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192", 'table_name_staging')
917
- "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name_staging', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
918
- >>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)", 'sales_product_rank_rt_replicated_2_staging')
919
- "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2_staging', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)"
920
- >>> engine_patch_replicated_engine("ReplicatedMergeTree", None, 't_000') is None
921
- True
922
- >>> engine_patch_replicated_engine("Log", "Log()", 't_000')
923
- 'Log()'
924
- >>> engine_patch_replicated_engine("MergeTree", "MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024", 't_000')
925
- 'MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
926
- """
927
- if not engine_full:
928
- return None
929
- if engine.lower().startswith("Replicated".lower()):
930
- parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
931
- paths = parts[2].split("/")
932
- paths[-1] = new_table_name
933
- zoo_path = "/".join(paths)
934
- return "".join([*parts[:2], zoo_path, *parts[3:]])
935
- return engine_full
936
-
937
-
938
914
  if __name__ == "__main__":
939
915
  print( # noqa: T201
940
916
  _parse_table_structure(
@@ -1396,7 +1396,7 @@ _namespace = {
1396
1396
  }
1397
1397
 
1398
1398
 
1399
- reserved_vars = set(["_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))])
1399
+ reserved_vars = {"_tt_tmp", "_tt_append", "isinstance", "str", "error", "custom_error", *list(vars(builtins))}
1400
1400
  for p in DEFAULT_PARAM_NAMES: # we handle these in an specific manner
1401
1401
  reserved_vars.discard(p) # `format` is part of builtins
1402
1402
  # Allow 'id' to be used as a template parameter - https://gitlab.com/tinybird/analytics/-/issues/19119
@@ -1542,35 +1542,6 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
1542
1542
  raise e
1543
1543
 
1544
1544
 
1545
- class CodeWriter:
1546
- def __init__(self, file, template):
1547
- self.file = file
1548
- self.current_template = template
1549
- self.apply_counter = 0
1550
- self._indent = 0
1551
-
1552
- def indent_size(self):
1553
- return self._indent
1554
-
1555
- def indent(self):
1556
- class Indenter:
1557
- def __enter__(_):
1558
- self._indent += 1
1559
- return self
1560
-
1561
- def __exit__(_, *args):
1562
- assert self._indent > 0
1563
- self._indent -= 1
1564
-
1565
- return Indenter()
1566
-
1567
- def write_line(self, line, line_number, indent=None):
1568
- if indent is None:
1569
- indent = self._indent
1570
- line_comment = " # %s:%d" % ("<generated>", line_number)
1571
- print(" " * indent + line + line_comment, file=self.file)
1572
-
1573
-
1574
1545
  def get_var_names(t: Template):
1575
1546
  """
1576
1547
  Extract variable names from a template.
@@ -1982,11 +1953,12 @@ def get_var_data(content, node_id=None):
1982
1953
  """
1983
1954
 
1984
1955
  def node_to_value(x):
1985
- if type(x) in (ast.Bytes, ast.Str):
1986
- return x.s
1987
- elif type(x) == ast.Num: # noqa: E721
1988
- return x.n
1989
- elif type(x) == ast.NameConstant: # noqa: E721
1956
+ # ast.Constant supersedes the deprecated ast.Bytes/ast.Str/ast.Num/
1957
+ # ast.NameConstant node classes. Those were aliases of ast.Constant
1958
+ # since Python 3.8 (emitting DeprecationWarning on access in 3.12) and
1959
+ # were removed in Python 3.14. The parser only ever produces
1960
+ # ast.Constant nodes, so .value holds str/bytes/int/float/bool/None.
1961
+ if type(x) == ast.Constant: # noqa: E721
1990
1962
  return x.value
1991
1963
  elif type(x) == ast.Name: # noqa: E721
1992
1964
  return x.id
@@ -2000,8 +1972,6 @@ def get_var_data(content, node_id=None):
2000
1972
  if not r:
2001
1973
  r = node_to_value(x.right)
2002
1974
  return r
2003
- elif type(x) == ast.Constant: # noqa: E721
2004
- return x.value
2005
1975
  elif type(x) == ast.UnaryOp and type(x.operand) == ast.Constant: # noqa: E721
2006
1976
  if type(x.op) == ast.USub: # noqa: E721
2007
1977
  return x.operand.value * -1
@@ -2020,11 +1990,9 @@ def get_var_data(content, node_id=None):
2020
1990
  return []
2021
1991
 
2022
1992
  first_elem = x.elts[0]
2023
- if type(first_elem) in (ast.Bytes, ast.Str):
2024
- return [elem.s for elem in x.elts]
2025
- elif type(first_elem) == ast.Num: # noqa: E721
2026
- return [elem.n for elem in x.elts]
2027
- elif type(first_elem) == ast.NameConstant or type(first_elem) == ast.Constant: # noqa: E721
1993
+ # See node_to_value: ast.Constant supersedes the deprecated
1994
+ # ast.Bytes/ast.Str/ast.Num/ast.NameConstant (removed in 3.14).
1995
+ if type(first_elem) == ast.Constant: # noqa: E721
2028
1996
  return [elem.value for elem in x.elts]
2029
1997
  elif type(first_elem) == ast.Name: # noqa: E721
2030
1998
  return [elem.id for elem in x.elts]
@@ -2295,7 +2263,12 @@ def get_var_names_and_types(t: Template, node_id: Optional[str] = None) -> List[
2295
2263
  vars_out.extend(
2296
2264
  parse_statement_code_if_new_vars(statement_code, skip_names=typed_names | statement_expr_names)
2297
2265
  )
2298
- return vars_out
2266
+ # Inside a {% %} statement a type function with a literal first arg
2267
+ # (e.g. {% set x = Float32(1.5) %}) is a plain value cast: the
2268
+ # assigned variable is the parameter, the literal is not. Drop
2269
+ # these artifacts so they don't surface as bogus params nor get
2270
+ # rejected by validate_template_parameter_names.
2271
+ return [vd for vd in vars_out if isinstance(vd["name"], str) and vd["name"]]
2299
2272
 
2300
2273
  def _n(chunks: list, vars_out: list[dict[str, Any]]) -> None:
2301
2274
  for x in chunks:
@@ -2342,6 +2315,43 @@ def get_var_names_and_types_cached(t: Template):
2342
2315
  return get_var_names_and_types(t)
2343
2316
 
2344
2317
 
2318
+ def validate_template_parameter_names(sql: str, node_id: Optional[str] = None) -> None:
2319
+ """Reject template parameters that don't have a usable name.
2320
+
2321
+ Type functions such as ``{{Float32(x)}}`` double as casts, so a literal
2322
+ first argument (e.g. ``{{Float32(1682892000.0)}}`` or ``{{String('')}}``)
2323
+ is parsed as a parameter whose name is a number or an empty string instead
2324
+ of an identifier. Such a "parameter" can never be supplied by a caller and
2325
+ breaks parameter metadata consumers, so it must be rejected at write time.
2326
+
2327
+ Raises ``ValueError`` with the same wording used elsewhere for invalid names.
2328
+
2329
+ Templates that fail to parse are left untouched: there are no parameters to
2330
+ validate, and reporting/deferring those errors is the job of the existing
2331
+ render/validation flow (some paths, e.g. ``ignore_sql_errors``, tolerate
2332
+ unparseable SQL on purpose). This keeps the check strictly additive. The
2333
+ only exception is the invalid-name ``ValueError`` that extraction itself
2334
+ raises (e.g. for a list literal used as first argument), which is exactly
2335
+ what this validator exists to report.
2336
+ """
2337
+ try:
2338
+ params = get_var_names_and_types(Template(sql), node_id=node_id)
2339
+ except SQLTemplateException:
2340
+ # Template-level problem (e.g. a SecurityException converted by
2341
+ # get_var_names_and_types), not an invalid parameter name.
2342
+ return
2343
+ except ValueError:
2344
+ # Extraction itself rejects some invalid names (e.g. a list literal
2345
+ # as first argument) with the same ValueError this function raises.
2346
+ raise
2347
+ except Exception:
2348
+ return
2349
+ for param in params:
2350
+ name = param.get("name")
2351
+ if not isinstance(name, str) or not name:
2352
+ raise ValueError(f'"{name}" can not be used as a variable name')
2353
+
2354
+
2345
2355
  def wrap_vars(t, escape_arrays: bool = False):
2346
2356
  def _n(chunks, v):
2347
2357
  for x in chunks:
@@ -2943,13 +2953,21 @@ def extract_variables_from_sql(sql: str, params: List[Dict[str, Any]]) -> Dict[s
2943
2953
  return defaults
2944
2954
 
2945
2955
 
2946
- def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict[str, str]] = None) -> str:
2956
+ def render_template_with_secrets(
2957
+ name: str,
2958
+ content: str,
2959
+ secrets: Optional[Dict[str, str]] = None,
2960
+ empty_secret_raises: bool = False,
2961
+ ) -> str:
2947
2962
  """Renders a template with secrets, allowing for default values.
2948
2963
 
2949
2964
  Args:
2950
2965
  name: The name of the template
2951
2966
  content: The template content
2952
2967
  secrets: A dictionary mapping secret names to their values
2968
+ empty_secret_raises: When True, empty secret values or empty defaults raise
2969
+ instead of returning '""'. Used for sink metadata (Forward deploy) where
2970
+ an empty bucket path must not be baked as s3://"".
2953
2971
 
2954
2972
  Returns:
2955
2973
  The rendered template
@@ -2991,6 +3009,26 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
2991
3009
  Traceback (most recent call last):
2992
3010
  ...
2993
3011
  tinybird.sql_template.SQLTemplateException: Template Syntax Error: Cannot access secret 'MISSING_SECRET'. Check the secret exists in the Workspace and the token has the required scope.
3012
+
3013
+ >>> render_template_with_secrets(
3014
+ ... "sink_bucket",
3015
+ ... "s3://{{ tb_secret('MISSING_SECRET', '') }}",
3016
+ ... secrets = {},
3017
+ ... empty_secret_raises=True,
3018
+ ... )
3019
+ Traceback (most recent call last):
3020
+ ...
3021
+ tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'MISSING_SECRET' resolves to an empty value and cannot be used in sink metadata.
3022
+
3023
+ >>> render_template_with_secrets(
3024
+ ... "sink_bucket",
3025
+ ... "s3://{{ tb_secret('EMPTY_SECRET', 'fallback') }}",
3026
+ ... secrets = {'EMPTY_SECRET': ''},
3027
+ ... empty_secret_raises=True,
3028
+ ... )
3029
+ Traceback (most recent call last):
3030
+ ...
3031
+ tinybird.sql_template.SQLTemplateException: Template Syntax Error: Secret 'EMPTY_SECRET' resolves to an empty value and cannot be used in sink metadata.
2994
3032
  """
2995
3033
  if not secrets:
2996
3034
  secrets = {}
@@ -3006,15 +3044,24 @@ def render_template_with_secrets(name: str, content: str, secrets: Optional[Dict
3006
3044
  The secret value or default
3007
3045
 
3008
3046
  Raises:
3009
- SQLTemplateException: If the secret is not found and no default is provided
3047
+ SQLTemplateException: If the secret is not found and no default is provided,
3048
+ or if empty_secret_raises is True and the resolved value is empty
3010
3049
  """
3011
3050
  if secret_name in secrets:
3012
3051
  value = secrets[secret_name]
3013
3052
  if isinstance(value, str) and len(value) == 0:
3053
+ if empty_secret_raises:
3054
+ raise SQLTemplateException(
3055
+ f"Secret '{secret_name}' resolves to an empty value and cannot be used in sink metadata."
3056
+ )
3014
3057
  return '""'
3015
3058
  return value
3016
3059
  elif default is not None:
3017
3060
  if isinstance(default, str) and len(default) == 0:
3061
+ if empty_secret_raises:
3062
+ raise SQLTemplateException(
3063
+ f"Secret '{secret_name}' resolves to an empty value and cannot be used in sink metadata."
3064
+ )
3018
3065
  return '""'
3019
3066
  return default
3020
3067
  else:
@@ -158,7 +158,20 @@ def has_unoptimized_join(sql: str, left_table: Optional[Union[Tuple[str, str], T
158
158
  raise UnoptimizedJoinException(sql)
159
159
 
160
160
 
161
- def format_where_for_mutation_command(where_clause: str, lightweight: bool = False) -> str:
161
+ def _format_fragment_for_mutation_command(fragment: str) -> str:
162
+ """Normalize a SQL fragment the way CH stores it in the
163
+ `system.mutations.command` column: format the expression with CH's own
164
+ formatter, then escape it as it appears inside the serialized command
165
+ (string literals keep backslash-escaped quotes)."""
166
+ formatted = chquery.format(f"""SELECT {fragment}""").split("SELECT ")[1]
167
+ formatted = formatted.replace("\\", "\\\\").replace("'", "''")
168
+ quoted = chquery.format(f"SELECT '{formatted}'").split("SELECT ")[1]
169
+ return quoted[1:-1]
170
+
171
+
172
+ def format_where_for_mutation_command(
173
+ where_clause: str, lightweight: bool = False, partition: Optional[str] = None
174
+ ) -> str:
162
175
  """
163
176
  >>> format_where_for_mutation_command("numnights = 99")
164
177
  'DELETE WHERE numnights = 99'
@@ -172,12 +185,18 @@ def format_where_for_mutation_command(where_clause: str, lightweight: bool = Fal
172
185
  "DELETE WHERE reservationid = \\\\'\\\\\\\\\\\\'foo\\\\'"
173
186
  >>> format_where_for_mutation_command("number < 3", lightweight=True)
174
187
  'UPDATE _row_exists = 0 WHERE number < 3'
188
+ >>> format_where_for_mutation_command("number < 3", lightweight=True, partition="201901")
189
+ 'UPDATE _row_exists = 0 IN PARTITION 201901 WHERE number < 3'
190
+ >>> format_where_for_mutation_command("number < 3", lightweight=True, partition="'2019-01-01'")
191
+ "UPDATE _row_exists = 0 IN PARTITION \\\\'2019-01-01\\\\' WHERE number < 3"
175
192
  """
176
- formatted_condition = chquery.format(f"""SELECT {where_clause}""").split("SELECT ")[1]
177
- formatted_condition = formatted_condition.replace("\\", "\\\\").replace("'", "''")
178
- quoted_condition = chquery.format(f"SELECT '{formatted_condition}'").split("SELECT ")[1]
179
- prefix = "UPDATE _row_exists = 0 WHERE" if lightweight else "DELETE WHERE"
180
- return f"{prefix} {quoted_condition[1:-1]}"
193
+ quoted_condition = _format_fragment_for_mutation_command(where_clause)
194
+ prefix = "UPDATE _row_exists = 0" if lightweight else "DELETE"
195
+ # Partition-scoped mutations store the partition expression in the
196
+ # command (`… IN PARTITION <expr> WHERE <cond>`), normalized by the same
197
+ # CH formatter, so it must be part of the match.
198
+ partition_clause = f" IN PARTITION {_format_fragment_for_mutation_command(partition)}" if partition else ""
199
+ return f"{prefix}{partition_clause} WHERE {quoted_condition}"
181
200
 
182
201
 
183
202
  # Functions that take table/dictionary names as string literal arguments.
@@ -624,7 +643,7 @@ def replace_tables(
624
643
 
625
644
  if current_replacements:
626
645
  # We need to transform the dictionary into something cacheable, so a sorted tuple of tuples it is
627
- r = tuple(sorted([(k, v) for k, v in current_replacements.items()]))
646
+ r = tuple(sorted(current_replacements.items()))
628
647
  sql = replace_tables_chquery_cached(
629
648
  sql,
630
649
  r,
@@ -590,7 +590,6 @@ async def regression_tests(
590
590
 
591
591
  if current_main_workspace["id"] == config["id"]:
592
592
  raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
593
- return
594
593
  try:
595
594
  response = await client.branch_regression_tests_file(
596
595
  config["id"], regression_tests_commands, run_in_main=main
@@ -175,11 +175,6 @@ def _get_current_workspace_common(
175
175
  return next((workspace for workspace in workspaces if workspace["id"] == current_workspace_id), None)
176
176
 
177
177
 
178
- async def get_current_environment(client, config):
179
- workspaces: List[Dict[str, Any]] = (await client.user_workspaces_and_branches()).get("workspaces", [])
180
- return next((workspace for workspace in workspaces if workspace["id"] == config["id"]), None)
181
-
182
-
183
178
  async def get_current_workspace_branches(config: CLIConfig) -> List[Dict[str, Any]]:
184
179
  current_main_workspace: Optional[Dict[str, Any]] = await get_current_main_workspace(config)
185
180
  if not current_main_workspace:
@@ -508,12 +503,6 @@ def format_host(host: str, subdomain: Optional[str] = None) -> str:
508
503
  return host
509
504
 
510
505
 
511
- def region_from_host(region_name_or_host, regions):
512
- """Returns the region that matches region_name_or_host"""
513
-
514
- return next((r for r in regions if _compare_region_host(region_name_or_host, r)), None)
515
-
516
-
517
506
  def ask_for_user_token(action: str, ui_host: str) -> str:
518
507
  return click.prompt(
519
508
  f'\nUse the token called "user token" in order to {action}. Copy it from {ui_host}/tokens and paste it here',
@@ -974,7 +963,8 @@ async def push_data(
974
963
  sql_condition: Optional[str] = None,
975
964
  replace_options=None,
976
965
  concurrency: int = 1,
977
- ):
966
+ use_v1: bool = False,
967
+ ) -> Optional[list[str]]:
978
968
  if url and type(url) is tuple:
979
969
  url = url[0]
980
970
  client: TinyB = ctx.obj["client"]
@@ -1011,6 +1001,8 @@ async def push_data(
1011
1001
  datasource_name: str, url: str, mode: str, sql_condition: Optional[str], replace_options: Optional[Set[str]]
1012
1002
  ):
1013
1003
  parsed = urlparse(url)
1004
+ if use_v1 and parsed.scheme in ("http", "https"):
1005
+ raise CLIException("--experimental=use_v1 only supports local files.")
1014
1006
  # poor man's format detection
1015
1007
  _format = get_format_from_filename_or_url(url)
1016
1008
  if parsed.scheme in ("http", "https"):
@@ -1031,8 +1023,15 @@ async def push_data(
1031
1023
  sql_condition=sql_condition,
1032
1024
  format=_format,
1033
1025
  replace_options=replace_options,
1026
+ use_v1=use_v1,
1034
1027
  )
1035
1028
 
1029
+ if use_v1:
1030
+ job_id = res.get("id") or res.get("import_id")
1031
+ if not isinstance(job_id, str):
1032
+ raise CLIException("The v1 import response did not include a job ID.")
1033
+ return job_id
1034
+
1036
1035
  datasource_name = res["datasource"]["name"]
1037
1036
  try:
1038
1037
  datasource = await client.get_datasource(datasource_name)
@@ -1063,6 +1062,8 @@ async def push_data(
1063
1062
  try:
1064
1063
  tasks = [process_url(datasource_name, url, mode, sql_condition, replace_options) for url in urls]
1065
1064
  output = await gather_with_concurrency(concurrency, *tasks)
1065
+ if use_v1:
1066
+ return list(output)
1066
1067
  parser, total_rows, appended_rows = list(output)[-1]
1067
1068
  except AuthNoTokenException:
1068
1069
  raise
@@ -1083,6 +1084,8 @@ async def push_data(
1083
1084
  click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
1084
1085
  click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
1085
1086
 
1087
+ return None
1088
+
1086
1089
 
1087
1090
  async def sync_data(ctx, datasource_name: str, yes: bool):
1088
1091
  client: TinyB = ctx.obj["client"]
@@ -1121,11 +1124,6 @@ def validate_datasource_name(name):
1121
1124
  raise CLIException(FeedbackManager.error_datasource_name())
1122
1125
 
1123
1126
 
1124
- def validate_connection_id(connection_id):
1125
- if not isinstance(connection_id, str) or connection_id == "":
1126
- raise CLIException(FeedbackManager.error_datasource_connection_id())
1127
-
1128
-
1129
1127
  def validate_kafka_topic(topic):
1130
1128
  if not isinstance(topic, str):
1131
1129
  raise CLIException(FeedbackManager.error_kafka_topic())
@@ -6,8 +6,6 @@ from enum import Enum
6
6
  from typing import Any, Dict, List, Optional, Tuple, Union
7
7
  from urllib.parse import urlparse
8
8
 
9
- from packaging import version
10
-
11
9
  import tinybird.client as tbc
12
10
  from tinybird.config import CURRENT_VERSION, DEFAULT_API_HOST, DEFAULT_LOCALHOST
13
11
 
@@ -34,12 +32,6 @@ class FeatureFlags:
34
32
  return not ("x.y.z" in CURRENT_VERSION and os.environ.get("TB_CLI_TELEMETRY_SEND_IN_LOCAL", "0") == "0")
35
33
 
36
34
 
37
- def compare_versions(a: str, b: str) -> int:
38
- va = version.parse(a)
39
- vb = version.parse(b)
40
- return -1 if va < vb else (1 if va > vb else 0)
41
-
42
-
43
35
  class ConfigValueOrigin(Enum):
44
36
  # Sources for config values (environment variables, .tinyb file or default value)
45
37
 
@@ -96,7 +88,7 @@ class CLIConfig:
96
88
  def to_dict(self) -> Dict[str, Any]:
97
89
  """Helper to ease"""
98
90
  result: Dict[str, Any] = self._parent.to_dict() if self._parent else {}
99
- result.update(dict((v.name, deepcopy(v.value)) for v in self._values.values()))
91
+ result.update({v.name: deepcopy(v.value) for v in self._values.values()})
100
92
  return result
101
93
 
102
94
  def __getitem__(self, key) -> Any:
@@ -179,12 +171,6 @@ class CLIConfig:
179
171
  def set_semver(self, semver: Optional[str]) -> None:
180
172
  self["semver"] = semver
181
173
 
182
- def get_semver(self) -> Optional[str]:
183
- try:
184
- return self["semver"]
185
- except KeyError:
186
- return None
187
-
188
174
  def set_token_for_host(self, token: Optional[str], host: Optional[str]) -> None:
189
175
  """Sets the token for the specified host.
190
176
 
@@ -15,7 +15,7 @@ from click import Context
15
15
 
16
16
  from tinybird.client import AuthNoTokenException, CanNotBeDeletedException, DoesNotExistException, TinyB
17
17
  from tinybird.config import get_display_host
18
- from tinybird.datafile_common import get_name_version, wait_job
18
+ from tinybird.datafile_common import get_name_version
19
19
  from tinybird.feedback_manager import FeedbackManager
20
20
  from tinybird.tb_cli_modules.branch import warn_if_in_live
21
21
  from tinybird.tb_cli_modules.cli import cli
@@ -34,9 +34,12 @@ from tinybird.tb_cli_modules.common import (
34
34
  validate_kafka_auto_offset_reset,
35
35
  validate_kafka_group,
36
36
  validate_kafka_topic,
37
+ wait_job,
37
38
  )
38
39
  from tinybird.tb_cli_modules.config import CLIConfig
39
- from tinybird.tb_cli_modules.exceptions import CLIDatasourceException
40
+ from tinybird.tb_cli_modules.exceptions import CLIDatasourceException, CLIException
41
+
42
+ EXPERIMENTAL_FEATURE_USE_V1 = "use_v1"
40
43
 
41
44
 
42
45
  @cli.group()
@@ -45,6 +48,34 @@ def datasource(ctx):
45
48
  """Data Sources commands"""
46
49
 
47
50
 
51
+ def _echo_v1_import_jobs_queued(job_ids: list[str], operation: str) -> None:
52
+ for job_id in job_ids:
53
+ click.echo(FeedbackManager.success_import_job_queued(operation=operation, job_id=job_id))
54
+ click.echo(FeedbackManager.info_import_job_status(job_id=job_id))
55
+
56
+
57
+ async def _wait_for_v1_import_jobs(client: TinyB, job_ids: list[str], operation: str) -> None:
58
+ for job_id in job_ids:
59
+ try:
60
+ await wait_job(client, job_id, f"/v0/jobs/{job_id}", f"{operation} import")
61
+ except CLIException:
62
+ if await _echo_v1_import_job_details(client, job_id):
63
+ raise click.exceptions.Exit(1)
64
+ raise
65
+ click.echo(FeedbackManager.success_import_job_completed(operation=operation, job_id=job_id))
66
+
67
+
68
+ async def _echo_v1_import_job_details(client: TinyB, job_id: str) -> bool:
69
+ try:
70
+ job = await client.job(job_id)
71
+ except Exception:
72
+ return False
73
+ click.echo(FeedbackManager.info_job(job=job_id))
74
+ echo_safe_humanfriendly_tables_format_smart_table([job.values()], column_names=job.keys())
75
+ click.echo("\n")
76
+ return True
77
+
78
+
48
79
  @datasource.command(name="ls")
49
80
  @click.option("--match", default=None, help="Retrieve any resources matching the pattern. eg --match _test")
50
81
  @click.option(
@@ -135,6 +166,13 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
135
166
  hidden=True,
136
167
  )
137
168
  @click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
169
+ @click.option("--wait", is_flag=True, default=False, help="Wait for a v1 import job to finish.")
170
+ @click.option(
171
+ "--experimental",
172
+ type=click.Choice([EXPERIMENTAL_FEATURE_USE_V1]),
173
+ multiple=True,
174
+ help="Enable an experimental feature. May be specified multiple times.",
175
+ )
138
176
  @click.pass_context
139
177
  @coro
140
178
  async def datasource_append(
@@ -145,6 +183,8 @@ async def datasource_append(
145
183
  incremental: Optional[str],
146
184
  ignore_empty: bool,
147
185
  concurrency: int,
186
+ experimental: tuple[str, ...],
187
+ wait: bool,
148
188
  ):
149
189
  """
150
190
  Appends data to an existing Data Source from URL, local file or a connector
@@ -157,7 +197,14 @@ async def datasource_append(
157
197
 
158
198
  if not url:
159
199
  raise CLIDatasourceException(FeedbackManager.error_missing_url(datasource=datasource_name))
160
- await push_data(ctx, datasource_name, url, mode="append", concurrency=concurrency)
200
+ use_v1 = EXPERIMENTAL_FEATURE_USE_V1 in experimental
201
+ if wait and not use_v1:
202
+ raise CLIDatasourceException("--wait requires --experimental=use_v1.")
203
+ job_ids = await push_data(ctx, datasource_name, url, mode="append", concurrency=concurrency, use_v1=use_v1)
204
+ if use_v1:
205
+ _echo_v1_import_jobs_queued(job_ids or [], "Append")
206
+ if wait:
207
+ await _wait_for_v1_import_jobs(ctx.obj["client"], job_ids or [], "Append")
161
208
 
162
209
 
163
210
  @datasource.command(name="replace")
@@ -165,6 +212,13 @@ async def datasource_append(
165
212
  @click.argument("url", nargs=-1)
166
213
  @click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
167
214
  @click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
215
+ @click.option("--wait", is_flag=True, default=False, help="Wait for a v1 import job to finish.")
216
+ @click.option(
217
+ "--experimental",
218
+ type=click.Choice([EXPERIMENTAL_FEATURE_USE_V1]),
219
+ multiple=True,
220
+ help="Enable an experimental feature. May be specified multiple times.",
221
+ )
168
222
  @click.pass_context
169
223
  @coro
170
224
  async def datasource_replace(
@@ -173,6 +227,8 @@ async def datasource_replace(
173
227
  url,
174
228
  sql_condition,
175
229
  skip_incompatible_partition_key,
230
+ experimental: tuple[str, ...],
231
+ wait: bool,
176
232
  ):
177
233
  """
178
234
  Replaces the data in a data source from a URL, local file or a connector
@@ -189,14 +245,22 @@ async def datasource_replace(
189
245
  replace_options = set()
190
246
  if skip_incompatible_partition_key:
191
247
  replace_options.add("skip_incompatible_partition_key")
192
- await push_data(
248
+ use_v1 = EXPERIMENTAL_FEATURE_USE_V1 in experimental
249
+ if wait and not use_v1:
250
+ raise CLIDatasourceException("--wait requires --experimental=use_v1.")
251
+ job_ids = await push_data(
193
252
  ctx,
194
253
  datasource_name,
195
254
  url,
196
255
  mode="replace",
197
256
  sql_condition=sql_condition,
198
257
  replace_options=replace_options,
258
+ use_v1=use_v1,
199
259
  )
260
+ if use_v1:
261
+ _echo_v1_import_jobs_queued(job_ids or [], "Replace")
262
+ if wait:
263
+ await _wait_for_v1_import_jobs(ctx.obj["client"], job_ids or [], "Replace")
200
264
 
201
265
 
202
266
  @datasource.command(name="analyze")
@@ -313,12 +313,3 @@ def test_run_summary(results: List[TestSummaryResults], only_fail: bool = False,
313
313
  raise CLIException(FeedbackManager.error_some_data_validation_have_failed())
314
314
  if total_counts.get(ERROR, 0) > 0:
315
315
  raise CLIException(FeedbackManager.error_some_tests_have_errors())
316
-
317
-
318
- def get_bare_url(url: str) -> str:
319
- if url.startswith("http://"):
320
- return url[7:]
321
- elif url.startswith("https://"):
322
- return url[8:]
323
- else:
324
- return url
@@ -114,7 +114,6 @@ async def clear_workspace(ctx: Context, yes: bool, dry_run: bool) -> None:
114
114
  if config["id"] == workspace["id"]:
115
115
  if workspace.get("is_branch"):
116
116
  raise CLIWorkspaceException(FeedbackManager.error_not_allowed_in_branch(cli="tb"))
117
- return
118
117
  else:
119
118
  click.echo(FeedbackManager.info_current_workspace())
120
119
  table.append(
@@ -916,10 +916,10 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
916
916
 
917
917
  # Intermediate ("else", "elif", etc) blocks
918
918
  intermediate_blocks = {
919
- "else": set(["if", "for", "while", "try"]),
920
- "elif": set(["if"]),
921
- "except": set(["try"]),
922
- "finally": set(["try"]),
919
+ "else": {"if", "for", "while", "try"},
920
+ "elif": {"if"},
921
+ "except": {"try"},
922
+ "finally": {"try"},
923
923
  }
924
924
  allowed_parents = intermediate_blocks.get(operator)
925
925
 
@@ -997,7 +997,7 @@ def _parse(reader: _TemplateReader, template, in_block=None, in_loop=None):
997
997
 
998
998
  elif operator in ("break", "continue"):
999
999
  if not in_loop:
1000
- reader.raise_parse_error("%s outside %s block" % (operator, set(["for", "while"])))
1000
+ reader.raise_parse_error("%s outside %s block" % (operator, {"for", "while"}))
1001
1001
  body.chunks.append(_Statement(contents, line))
1002
1002
  continue
1003
1003
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird_cli
3
- Version: 6.5.4.dev0
3
+ Version: 6.5.5.dev0
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli
6
6
  Author: Tinybird
@@ -43,10 +43,19 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
43
43
  Changelog
44
44
  ----------
45
45
 
46
+ 6.5.4
47
+ ***********
48
+
49
+ - `Added` ``tb datasource append`` and ``tb datasource replace`` support the experimental ``--experimental=use_v1`` option for local CSV, NDJSON, and Parquet files. It submits the import through the v1 endpoint and prints the queued job ID; pass ``--wait`` to wait for completion.
50
+ - `Fixed` Internal improvements to secret template rendering used during datafile parsing.
51
+ - `Fixed` `tb push` now rejects pipes that declare a parameter without a valid name (e.g. a numeric-literal cast like ``{{Float32(1682892000.0)}}``), instead of creating an unusable parameter.
52
+
46
53
  6.5.3
47
54
  ***********
48
55
 
56
+ - `Removed` `bi_stats_rt` and `bi_stats` from service datasource catalogs and dependency ignore lists.
49
57
  - `Added` ``tb datasource delete`` now accepts ``--lightweight-delete``, ``--partition`` and ``--projection-mode``, backed by the ``POST /v1/datasources/{name}/delete`` endpoint that uses ClickHouse's lightweight ``DELETE FROM`` instead of the heavy ``ALTER TABLE ... DELETE`` mutation. Lightweight delete is synchronous by default and returns the affected row count plus a partition-level mutation block; pass ``--no-wait`` to enqueue a job and receive the v0-compatible envelope.
58
+ - `Changed` Upgraded Tornado to 6.5.5.
50
59
 
51
60
  6.5.1
52
61
  ***********