tinybird 4.5.2.dev0__tar.gz → 4.5.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/PKG-INFO +12 -1
  2. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/datafile/common.py +3 -2
  3. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/datatypes.py +2 -2
  4. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/service_datasources.py +1 -0
  5. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/sql.py +13 -7
  6. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/sql_template.py +1 -1
  7. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/sql_toolset.py +1 -19
  8. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/__cli__.py +2 -2
  9. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/branch.py +0 -1
  10. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/build.py +6 -0
  11. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/build_common.py +233 -97
  12. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/cicd.py +4 -4
  13. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/cli.py +5 -5
  14. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/common.py +10 -8
  15. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/create.py +0 -12
  16. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/build.py +2 -2
  17. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/build_datasource.py +1 -1
  18. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/build_pipe.py +1 -1
  19. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/playground.py +2 -2
  20. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/deployment.py +1 -4
  21. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/deployment_common.py +36 -20
  22. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/job.py +1 -3
  23. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/local.py +10 -11
  24. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/login_common.py +2 -4
  25. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/query_output.py +1 -3
  26. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/workspace.py +10 -11
  27. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/workspace_members.py +1 -3
  28. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/branch.py +0 -1
  29. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/common.py +0 -1
  30. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird.egg-info/PKG-INFO +12 -1
  31. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/setup.cfg +0 -0
  32. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/__cli__.py +0 -0
  33. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/ch_utils/constants.py +0 -0
  34. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/ch_utils/engine.py +0 -0
  35. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/check_pypi.py +0 -0
  36. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/client.py +0 -0
  37. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/config.py +0 -0
  38. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/context.py +0 -0
  39. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/datafile/exceptions.py +0 -0
  40. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/datafile/parse_connection.py +0 -0
  41. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/datafile/parse_datasource.py +0 -0
  42. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/datafile/parse_pipe.py +0 -0
  43. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/feedback_manager.py +0 -0
  44. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/git_settings.py +0 -0
  45. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/prompts.py +0 -0
  46. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/sql_template_fmt.py +0 -0
  47. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/syncasync.py +0 -0
  48. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/check_pypi.py +0 -0
  49. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/cli.py +0 -0
  50. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/client.py +0 -0
  51. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/config.py +0 -0
  52. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/config.py +0 -0
  53. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/connection.py +0 -0
  54. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/connection_kafka.py +0 -0
  55. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/connection_s3.py +0 -0
  56. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/copy.py +0 -0
  57. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/build_common.py +0 -0
  58. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/diff.py +0 -0
  59. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/fixture.py +0 -0
  60. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/format_common.py +0 -0
  61. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/format_connection.py +0 -0
  62. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  63. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  64. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  65. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datafile/pull.py +0 -0
  66. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/datasource.py +0 -0
  67. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/deprecations.py +0 -0
  68. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/endpoint.py +0 -0
  69. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/exceptions.py +0 -0
  70. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/feedback_manager.py +0 -0
  71. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/fmt.py +0 -0
  72. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/info.py +0 -0
  73. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/infra.py +0 -0
  74. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/job_common.py +0 -0
  75. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/llm.py +0 -0
  76. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/llm_utils.py +0 -0
  77. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/local_common.py +0 -0
  78. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/local_logs.py +0 -0
  79. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/login.py +0 -0
  80. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/logout.py +0 -0
  81. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/logs.py +0 -0
  82. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/materialization.py +0 -0
  83. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/open.py +0 -0
  84. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/pipe.py +0 -0
  85. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/preview.py +0 -0
  86. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/project.py +0 -0
  87. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/project_commands.py +0 -0
  88. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/py_project.py +0 -0
  89. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/regions.py +0 -0
  90. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/secret.py +0 -0
  91. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/secret_common.py +0 -0
  92. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/sink.py +0 -0
  93. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/table.py +0 -0
  94. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/telemetry.py +0 -0
  95. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/test.py +0 -0
  96. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/test_common.py +0 -0
  97. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  98. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  99. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/token.py +0 -0
  100. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/ts_project.py +0 -0
  101. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb/modules/watch.py +0 -0
  102. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli.py +0 -0
  103. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/auth.py +0 -0
  104. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/cicd.py +0 -0
  105. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/cli.py +0 -0
  106. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/config.py +0 -0
  107. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/connection.py +0 -0
  108. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/datasource.py +0 -0
  109. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/exceptions.py +0 -0
  110. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/fmt.py +0 -0
  111. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/job.py +0 -0
  112. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/pipe.py +0 -0
  113. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/regions.py +0 -0
  114. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/tag.py +0 -0
  115. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/telemetry.py +0 -0
  116. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/test.py +0 -0
  117. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  118. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  119. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/workspace.py +0 -0
  120. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  121. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird/tornado_template.py +0 -0
  122. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird.egg-info/SOURCES.txt +0 -0
  123. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird.egg-info/dependency_links.txt +0 -0
  124. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird.egg-info/entry_points.txt +0 -0
  125. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird.egg-info/requires.txt +0 -0
  126. {tinybird-4.5.2.dev0 → tinybird-4.5.3}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 4.5.2.dev0
3
+ Version: 4.5.3
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -52,6 +52,17 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
52
52
  Changelog
53
53
  ----------
54
54
 
55
+ 4.5.3
56
+ *******
57
+
58
+ - `Changed` `tb init` now defaults to CLI projects when `--type` is not provided.
59
+ - `Fixed` GitHub and GitLab CI templates generated by `tb init` now run `tb --local build` and `tb --local test run`.
60
+
61
+ 4.5.2
62
+ ***********
63
+
64
+ - `Fixed` `tb build` on branches now runs as an asynchronous job, avoiding timeouts on long-running executions.
65
+
55
66
  4.5.1
56
67
  *******
57
68
 
@@ -1394,7 +1394,8 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
1394
1394
  if match := _PATTERN_SIMPLE_AGG_FUNC.search(t):
1395
1395
  fn = match.group(1)
1396
1396
  inner_type = match.group(2)
1397
- result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
1397
+ if "Nullable(" not in inner_type:
1398
+ result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
1398
1399
  return result
1399
1400
 
1400
1401
 
@@ -2366,7 +2367,7 @@ def get_project_fixtures(folder: str) -> List[str]:
2366
2367
  def has_internal_datafiles(folder: str) -> bool:
2367
2368
  folder = folder or "."
2368
2369
  filenames = get_project_filenames(folder)
2369
- return any([f for f in filenames if "spans" in str(f) and "vendor" not in str(f)])
2370
+ return any(f for f in filenames if "spans" in str(f) and "vendor" not in str(f))
2370
2371
 
2371
2372
 
2372
2373
  def peek(iterable):
@@ -117,11 +117,11 @@ def date_test(x: str) -> bool:
117
117
 
118
118
 
119
119
  def datetime64_test(x: str) -> bool:
120
- return any([p.match(x) for p in datetime64_patterns])
120
+ return any(p.match(x) for p in datetime64_patterns)
121
121
 
122
122
 
123
123
  def datetime_test(x: str) -> bool:
124
- return any([p.match(x) for p in datetime_patterns])
124
+ return any(p.match(x) for p in datetime_patterns)
125
125
 
126
126
 
127
127
  def int_8_test(x: str) -> bool:
@@ -45,6 +45,7 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
45
45
  {"name": "method", "type": "String"},
46
46
  {"name": "release", "type": "String"},
47
47
  {"name": "user_agent", "type": "Nullable(String)"},
48
+ {"name": "client_ip", "type": "Nullable(String)"},
48
49
  {"name": "resource_tags", "type": "Array(String)"},
49
50
  {"name": "memory_usage", "type": "UInt64"},
50
51
  ],
@@ -233,10 +233,21 @@ def try_to_fix_nullable_in_simple_aggregating_function(t: str) -> Optional[str]:
233
233
  if match := _RE_TRY_FIX_NULLABLE_SAF.search(t):
234
234
  fn = match.group(1)
235
235
  inner_type = match.group(2)
236
- result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
236
+ if "Nullable(" not in inner_type:
237
+ result = f"SimpleAggregateFunction({fn}, Nullable({inner_type}))"
237
238
  return result
238
239
 
239
240
 
241
+ def wrap_nullable(col: dict[str, Any]):
242
+ if col["nullable"]:
243
+ if (col_type := try_to_fix_nullable_in_simple_aggregating_function(col["type"])) is None:
244
+ # Skip wrapping if Nullable already present, e.g. LowCardinality(Nullable(String))
245
+ col_type = col["type"] if "Nullable(" in col["type"] else "Nullable(%s)" % col["type"]
246
+ else:
247
+ col_type = col["type"]
248
+ return col_type
249
+
250
+
240
251
  def schema_to_sql_columns(schema: List[Dict[str, Any]], skip_jsonpaths: bool = False) -> List[str]:
241
252
  """return an array with each column in SQL
242
253
  >>> schema_to_sql_columns([{'name': 'temperature', 'type': 'Float32', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'temperature'}, {'name': 'temperature_delta', 'type': 'Float32', 'codec': 'CODEC(Delta(4), LZ4))', 'default_value': 'MATERIALIZED temperature', 'nullable': False, 'normalized_name': 'temperature_delta'}])
@@ -255,12 +266,7 @@ def schema_to_sql_columns(schema: List[Dict[str, Any]], skip_jsonpaths: bool = F
255
266
  columns: List[str] = []
256
267
  for x in schema:
257
268
  name = x["normalized_name"] if "normalized_name" in x else x["name"]
258
- if x["nullable"]:
259
- if (_type := try_to_fix_nullable_in_simple_aggregating_function(x["type"])) is None:
260
- # Skip wrapping if Nullable already present, e.g. LowCardinality(Nullable(String))
261
- _type = x["type"] if "Nullable(" in x["type"] else "Nullable(%s)" % x["type"]
262
- else:
263
- _type = x["type"]
269
+ _type = wrap_nullable(x)
264
270
  parts = [col_name(name, backquotes=True), _type]
265
271
  if x.get("jsonpath", None) and not skip_jsonpaths:
266
272
  parts.append(f"`json:{x['jsonpath']}`")
@@ -2826,7 +2826,7 @@ def render_sql_template(
2826
2826
  return Comment("error launched")
2827
2827
 
2828
2828
  v: dict = {x["name"]: Placeholder(x["name"], x["line"]) for x in template_variables}
2829
- is_tb_secret = any([s for s in template_variables if s["name"] == "tb_secret" or s["name"] == "tb_var"])
2829
+ is_tb_secret = any(s for s in template_variables if s["name"] == "tb_secret" or s["name"] == "tb_var")
2830
2830
 
2831
2831
  if variables:
2832
2832
  v.update(variables)
@@ -22,7 +22,7 @@ VALID_REMOTE = "VALID_REMOTE"
22
22
 
23
23
  class InvalidFunction(ValueError):
24
24
  def __init__(self, msg: str = "", table_function_name: str = ""):
25
- if any([fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg]):
25
+ if any(fn for fn in COPY_ENABLED_TABLE_FUNCTIONS if fn in msg):
26
26
  msg = msg.replace("is restricted", "is restricted to Copy Pipes")
27
27
 
28
28
  if table_function_name:
@@ -75,19 +75,6 @@ def explain_plan(sql: str) -> str:
75
75
  return chquery.explain_ast(sql)
76
76
 
77
77
 
78
- @dataclass(frozen=True)
79
- class ColumnInfo:
80
- name: str
81
- type: str
82
- nullable: bool
83
- default_specifier: str = ""
84
- default_expression: str | None = None
85
- codec: str | None = None
86
- comment: str | None = None
87
- ttl: str | None = None
88
- is_primary_key: bool = False
89
-
90
-
91
78
  @dataclass
92
79
  class MaterializedViewTarget:
93
80
  database: Optional[str]
@@ -118,11 +105,6 @@ def parse_materialized_view_target(create_table_query: str) -> Optional[Material
118
105
  )
119
106
 
120
107
 
121
- def get_columns_from_create_query(sql_schema: str) -> list[ColumnInfo]:
122
- columns = chquery.get_columns_from_create_query(sql_schema)
123
- return [ColumnInfo(**col) for col in columns]
124
-
125
-
126
108
  def has_join(sql: str) -> bool:
127
109
  return any(line.rstrip().startswith("TableJoin") for line in explain_plan(sql).split())
128
110
 
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/forward/commands'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '4.5.2.dev0'
8
- __revision__ = '4cf88c3'
7
+ __version__ = '4.5.3'
8
+ __revision__ = '23af8bd'
@@ -27,7 +27,6 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager, get_cli_name
27
27
  @cli.group()
28
28
  def branch() -> None:
29
29
  """Branch commands. Custom branches is an experimental feature in beta."""
30
- pass
31
30
 
32
31
 
33
32
  @branch.command(name="ls")
@@ -75,6 +75,7 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
75
75
  tb_client: TinyB = ctx.ensure_object(dict)["client"]
76
76
  config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
77
77
  is_branch = bool(ctx.ensure_object(dict)["branch"])
78
+ use_deployment_api = obj["env"] == "cloud" and is_branch
78
79
 
79
80
  # TODO: Explain that you can use custom branches too once they are open for everyone
80
81
  if obj["env"] == "cloud" and not is_branch:
@@ -100,6 +101,7 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
100
101
  config=config,
101
102
  is_branch=is_branch,
102
103
  with_connections=with_connections,
104
+ use_deployment_api=use_deployment_api,
103
105
  )
104
106
  if watch:
105
107
  run_watch(
@@ -113,6 +115,7 @@ def build(ctx: click.Context, watch: bool, with_connections: bool) -> None:
113
115
  config=config,
114
116
  is_branch=is_branch,
115
117
  with_connections=with_connections,
118
+ use_deployment_api=use_deployment_api,
116
119
  ),
117
120
  )
118
121
 
@@ -128,6 +131,7 @@ def dev(ctx: click.Context, with_connections: Optional[bool]) -> None:
128
131
  obj: Dict[str, Any] = ctx.ensure_object(dict)
129
132
  branch: Optional[str] = ctx.ensure_object(dict)["branch"]
130
133
  is_branch = bool(branch)
134
+ use_deployment_api = obj["env"] == "cloud" and is_branch
131
135
 
132
136
  # Default with_connections to True for branches, False otherwise
133
137
  if with_connections is None:
@@ -149,6 +153,7 @@ def dev(ctx: click.Context, with_connections: Optional[bool]) -> None:
149
153
  config=config,
150
154
  is_branch=is_branch,
151
155
  with_connections=with_connections,
156
+ use_deployment_api=use_deployment_api,
152
157
  )
153
158
  run_watch(
154
159
  project=project,
@@ -160,6 +165,7 @@ def dev(ctx: click.Context, with_connections: Optional[bool]) -> None:
160
165
  config=config,
161
166
  is_branch=is_branch,
162
167
  with_connections=with_connections,
168
+ use_deployment_api=use_deployment_api,
163
169
  ),
164
170
  )
165
171
 
@@ -14,6 +14,7 @@ from tinybird.datafile.parse_datasource import parse_datasource
14
14
  from tinybird.tb.client import TinyB
15
15
  from tinybird.tb.modules.common import push_data, sys_exit
16
16
  from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_dir, persist_fixture
17
+ from tinybird.tb.modules.deployment_common import api_fetch
17
18
  from tinybird.tb.modules.feedback_manager import FeedbackManager
18
19
  from tinybird.tb.modules.local_common import get_local_tokens
19
20
  from tinybird.tb.modules.project import Project
@@ -33,6 +34,7 @@ def process(
33
34
  project_with_vendors: Optional[Project] = None,
34
35
  is_branch: bool = False,
35
36
  with_connections: bool = False,
37
+ use_deployment_api: bool = False,
36
38
  ) -> Optional[str]:
37
39
  time_start = time.time()
38
40
 
@@ -65,6 +67,7 @@ def process(
65
67
  load_fixtures,
66
68
  project_with_vendors=project_with_vendors,
67
69
  with_connections=with_connections,
70
+ use_deployment_api=use_deployment_api,
68
71
  )
69
72
 
70
73
  except click.ClickException as e:
@@ -196,13 +199,8 @@ def build_project(
196
199
  load_fixtures: bool = True,
197
200
  project_with_vendors: Optional[Project] = None,
198
201
  with_connections: bool = False,
202
+ use_deployment_api: bool = False,
199
203
  ) -> Optional[bool]:
200
- MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
201
- DATAFILE_TYPE_TO_CONTENT_TYPE = {
202
- ".datasource": "text/plain",
203
- ".pipe": "text/plain",
204
- ".connection": "text/plain",
205
- }
206
204
  build_url = "/v1/build"
207
205
  if with_connections:
208
206
  build_url = f"{build_url}?with_connections=true"
@@ -213,28 +211,20 @@ def build_project(
213
211
  error: Optional[str] = None
214
212
 
215
213
  try:
216
- files = [
217
- ("context://", ("cli-version", "1.0.0", "text/plain")),
218
- ]
219
- project_path = project.path
220
- project_files = project.get_project_files()
214
+ if use_deployment_api:
215
+ return build_project_with_deploy_api(
216
+ project=project,
217
+ tb_client=tb_client,
218
+ silent=silent,
219
+ load_fixtures=load_fixtures,
220
+ project_with_vendors=project_with_vendors,
221
+ )
222
+
223
+ files, project_files = get_build_request_files(project, project_with_vendors)
221
224
 
222
225
  if not project_files:
223
226
  return False
224
227
 
225
- for file_path in project_files:
226
- relative_path = Path(file_path).relative_to(project_path).as_posix()
227
- with open(file_path, "rb") as fd:
228
- content_type = DATAFILE_TYPE_TO_CONTENT_TYPE.get(Path(file_path).suffix, "application/unknown")
229
- content = fd.read().decode("utf-8")
230
- if project_with_vendors:
231
- # Replace 'SHARED_WITH' and everything that comes after, including new lines, with 'SHARED_WITH Tinybird_Local_Test_'
232
- content = replace_shared_with(
233
- content,
234
- [project_with_vendors.workspace_name],
235
- )
236
-
237
- files.append((MULTIPART_BOUNDARY_DATA_PROJECT, (relative_path, content, content_type)))
238
228
  HEADERS = {"Authorization": f"Bearer {TINYBIRD_API_KEY}"}
239
229
  params = {"from": request_from} if request_from else None
240
230
  r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS, params=params)
@@ -249,90 +239,236 @@ def build_project(
249
239
 
250
240
  build_result = result.get("result")
251
241
  if build_result == "success":
252
- build = result.get("build")
253
- new_datasources = build.get("new_datasource_names", [])
254
- new_pipes = build.get("new_pipe_names", [])
255
- new_connections = build.get("new_data_connector_names", [])
256
- changed_datasources = build.get("changed_datasource_names", [])
257
- changed_pipes = build.get("changed_pipe_names", [])
258
- changed_connections = build.get("changed_data_connector_names", [])
259
- deleted_datasources = build.get("deleted_datasource_names", [])
260
- deleted_pipes = build.get("deleted_pipe_names", [])
261
- deleted_connections = build.get("deleted_data_connector_names", [])
262
-
263
- no_changes = (
264
- not new_datasources
265
- and not changed_datasources
266
- and not new_pipes
267
- and not changed_pipes
268
- and not new_connections
269
- and not changed_connections
270
- and not deleted_datasources
271
- and not deleted_pipes
272
- and not deleted_connections
273
- )
274
- if no_changes:
242
+ build = result.get("build") or {}
243
+ changes = get_build_changes(build)
244
+ if not has_build_changes(changes):
275
245
  return False
276
- if not silent:
277
- echo_changes(project, new_datasources, ".datasource", "created")
278
- echo_changes(project, changed_datasources, ".datasource", "changed")
279
- echo_changes(project, deleted_datasources, ".datasource", "deleted")
280
- echo_changes(project, new_pipes, ".pipe", "created")
281
- echo_changes(project, changed_pipes, ".pipe", "changed")
282
- echo_changes(project, deleted_pipes, ".pipe", "deleted")
283
- echo_changes(project, new_connections, ".connection", "created")
284
- echo_changes(project, changed_connections, ".connection", "changed")
285
- echo_changes(project, deleted_connections, ".connection", "deleted")
246
+ echo_build_changes(project, changes, silent)
286
247
  if load_fixtures:
287
- try:
288
- for filename in project_files:
289
- if filename.endswith(".datasource"):
290
- ds_path = Path(filename)
291
- ds_name = ds_path.stem
292
- fixture_folder = get_fixture_dir(project.folder)
293
- fixture_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
294
- fixture_path = next(
295
- (
296
- fixture_folder / f"{ds_name}{ext}"
297
- for ext in fixture_extensions
298
- if (fixture_folder / f"{ds_name}{ext}").exists()
299
- ),
300
- None,
301
- )
302
- if not fixture_path:
303
- sql_path = fixture_folder / f"{ds_name}.sql"
304
- if sql_path.exists():
305
- fixture_path = rebuild_fixture_sql(project, tb_client, str(sql_path))
306
-
307
- if fixture_path:
308
- append_fixture(tb_client, ds_name, str(fixture_path))
309
-
310
- except Exception as e:
311
- click.echo(FeedbackManager.error_exception(error=f"Error appending fixtures for '{ds_name}': {e}"))
312
-
313
- feedback = build.get("feedback", [])
314
- for f in feedback:
315
- click.echo(
316
- FeedbackManager.warning(message=f"△ {f.get('level')}: {f.get('resource')}: {f.get('message')}")
317
- )
248
+ append_project_fixtures(project, tb_client, project_files)
249
+ echo_build_feedback(build.get("feedback", []))
250
+ return True
318
251
  elif build_result == "failed":
319
- build_errors = result.get("errors")
320
- full_error_msg = ""
321
- for build_error in build_errors:
322
- filename_bit = build_error.get("filename", build_error.get("resource", ""))
323
- error_bit = build_error.get("error") or build_error.get("message") or ""
324
- error_msg = ((filename_bit + "\n") if filename_bit else "") + error_bit
325
- full_error_msg += error_msg + "\n\n"
326
- error = full_error_msg.strip("\n") or "Unknown build error"
252
+ error = format_build_errors(result.get("errors", []))
327
253
  else:
328
254
  error = f"Unknown build result. Error: {result.get('error')}"
255
+ except click.ClickException:
256
+ raise
329
257
  except Exception as e:
330
258
  error = str(e)
331
259
 
332
260
  if error:
333
261
  raise click.ClickException(error)
334
262
 
335
- return build_result
263
+ return False
264
+
265
+
266
+ def build_project_with_deploy_api(
267
+ project: Project,
268
+ tb_client: TinyB,
269
+ silent: bool = False,
270
+ load_fixtures: bool = True,
271
+ project_with_vendors: Optional[Project] = None,
272
+ ) -> Optional[bool]:
273
+ deploy_url = urljoin(tb_client.host, "/v1/deploy")
274
+ logging.debug(deploy_url)
275
+ request_from = getattr(tb_client, "request_from", None)
276
+ files, project_files = get_build_request_files(project, project_with_vendors)
277
+
278
+ if not project_files:
279
+ return False
280
+
281
+ headers = {"Authorization": f"Bearer {tb_client.token}"}
282
+ params: dict[str, str] = {"auto_promote": "true"}
283
+ if request_from:
284
+ params["from"] = request_from
285
+
286
+ response = requests.post(deploy_url, files=files, headers=headers, params=params)
287
+ try:
288
+ result = response.json()
289
+ except Exception as e:
290
+ logging.debug(e, exc_info=True)
291
+ click.echo(FeedbackManager.error(message="Couldn't parse response from server"))
292
+ sys_exit("build_error", str(e))
293
+
294
+ logging.debug(json.dumps(result, indent=2))
295
+
296
+ build_result = result.get("result")
297
+ deployment = result.get("deployment") or {}
298
+ if build_result == "no_changes":
299
+ return False
300
+ if build_result != "success":
301
+ deployment_errors = deployment.get("errors", []) if deployment else result.get("errors", [])
302
+ raise click.ClickException(result.get("error") or format_build_errors(deployment_errors))
303
+ if not deployment:
304
+ raise click.ClickException("Couldn't parse deployment response from server")
305
+
306
+ deployment = wait_for_build_deployment_to_be_live(
307
+ tb_client=tb_client,
308
+ headers=headers,
309
+ deployment=deployment,
310
+ request_from=request_from,
311
+ silent=silent,
312
+ )
313
+ changes = get_build_changes(deployment)
314
+ if not has_build_changes(changes):
315
+ return False
316
+
317
+ echo_build_changes(project, changes, silent)
318
+ if load_fixtures:
319
+ append_project_fixtures(project, tb_client, project_files)
320
+ echo_build_feedback(deployment.get("feedback", []))
321
+ return True
322
+
323
+
324
+ def get_build_request_files(
325
+ project: Project,
326
+ project_with_vendors: Optional[Project] = None,
327
+ ) -> tuple[list[tuple[str, tuple[str, str, str]]], list[str]]:
328
+ multipart_boundary_data_project = "data_project://"
329
+ datafile_type_to_content_type = {
330
+ ".datasource": "text/plain",
331
+ ".pipe": "text/plain",
332
+ ".connection": "text/plain",
333
+ }
334
+ files: list[tuple[str, tuple[str, str, str]]] = [
335
+ ("context://", ("cli-version", "1.0.0", "text/plain")),
336
+ ]
337
+ project_path = project.path
338
+ project_files = project.get_project_files()
339
+
340
+ for file_path in project_files:
341
+ relative_path = Path(file_path).relative_to(project_path).as_posix()
342
+ with open(file_path, "rb") as fd:
343
+ content_type = datafile_type_to_content_type.get(Path(file_path).suffix, "application/unknown")
344
+ content = fd.read().decode("utf-8")
345
+ if project_with_vendors:
346
+ # Replace SHARED_WITH targets when building vendored workspaces against the main project.
347
+ content = replace_shared_with(content, [project_with_vendors.workspace_name])
348
+
349
+ files.append((multipart_boundary_data_project, (relative_path, content, content_type)))
350
+
351
+ return files, project_files
352
+
353
+
354
+ def wait_for_build_deployment_to_be_live(
355
+ tb_client: TinyB,
356
+ headers: dict[str, str],
357
+ deployment: dict[str, Any],
358
+ request_from: Optional[str],
359
+ silent: bool,
360
+ ) -> dict[str, Any]:
361
+ if not silent:
362
+ click.echo(FeedbackManager.highlight(message="» Waiting for deployment to be ready..."))
363
+
364
+ poll_interval = 5
365
+ times_seen_failed = 0
366
+ while True:
367
+ url = f"{tb_client.host}/v1/deployments/{deployment.get('id')}"
368
+ result = api_fetch(url, headers, request_from=request_from)
369
+ deployment = result.get("deployment", {})
370
+ if not deployment:
371
+ raise click.ClickException("Error parsing deployment from response")
372
+
373
+ status = deployment.get("status")
374
+ if status == "failed":
375
+ times_seen_failed += 1
376
+ if times_seen_failed > 60:
377
+ raise click.ClickException("Deployment failed and wasn't deleted automatically")
378
+ time.sleep(poll_interval)
379
+ continue
380
+
381
+ if status in ("deleting", "deleted"):
382
+ errors = deployment.get("errors", [])
383
+ raise click.ClickException(f"Deployment deleted after failure. Errors: {errors}")
384
+
385
+ if deployment.get("live"):
386
+ return deployment
387
+
388
+ time.sleep(poll_interval)
389
+
390
+
391
+ def get_build_changes(result: dict[str, Any]) -> dict[str, list[str]]:
392
+ return {
393
+ "new_datasources": result.get("new_datasource_names", []),
394
+ "changed_datasources": result.get("changed_datasource_names", []),
395
+ "deleted_datasources": result.get("deleted_datasource_names", []),
396
+ "new_pipes": result.get("new_pipe_names", []),
397
+ "changed_pipes": result.get("changed_pipe_names", []),
398
+ "deleted_pipes": result.get("deleted_pipe_names", []),
399
+ "new_connections": result.get("new_data_connector_names", []),
400
+ "changed_connections": result.get("changed_data_connector_names", []),
401
+ "deleted_connections": result.get("deleted_data_connector_names", []),
402
+ }
403
+
404
+
405
+ def has_build_changes(changes: dict[str, list[str]]) -> bool:
406
+ return any(changes.values())
407
+
408
+
409
+ def echo_build_changes(project: Project, changes: dict[str, list[str]], silent: bool) -> None:
410
+ if silent:
411
+ return
412
+
413
+ echo_changes(project, changes["new_datasources"], ".datasource", "created")
414
+ echo_changes(project, changes["changed_datasources"], ".datasource", "changed")
415
+ echo_changes(project, changes["deleted_datasources"], ".datasource", "deleted")
416
+ echo_changes(project, changes["new_pipes"], ".pipe", "created")
417
+ echo_changes(project, changes["changed_pipes"], ".pipe", "changed")
418
+ echo_changes(project, changes["deleted_pipes"], ".pipe", "deleted")
419
+ echo_changes(project, changes["new_connections"], ".connection", "created")
420
+ echo_changes(project, changes["changed_connections"], ".connection", "changed")
421
+ echo_changes(project, changes["deleted_connections"], ".connection", "deleted")
422
+
423
+
424
+ def append_project_fixtures(
425
+ project: Project,
426
+ tb_client: TinyB,
427
+ project_files: list[str],
428
+ ) -> None:
429
+ ds_name = ""
430
+ try:
431
+ for filename in project_files:
432
+ if not filename.endswith(".datasource"):
433
+ continue
434
+
435
+ ds_name = Path(filename).stem
436
+ fixture_folder = get_fixture_dir(project.folder)
437
+ fixture_extensions = [FixtureExtension.NDJSON, FixtureExtension.CSV]
438
+ fixture_path = next(
439
+ (
440
+ fixture_folder / f"{ds_name}{ext}"
441
+ for ext in fixture_extensions
442
+ if (fixture_folder / f"{ds_name}{ext}").exists()
443
+ ),
444
+ None,
445
+ )
446
+ if not fixture_path:
447
+ sql_path = fixture_folder / f"{ds_name}.sql"
448
+ if sql_path.exists():
449
+ fixture_path = rebuild_fixture_sql(project, tb_client, str(sql_path))
450
+
451
+ if fixture_path:
452
+ append_fixture(tb_client, ds_name, str(fixture_path))
453
+ except Exception as e:
454
+ click.echo(FeedbackManager.error_exception(error=f"Error appending fixtures for '{ds_name}': {e}"))
455
+
456
+
457
+ def echo_build_feedback(feedback: list[dict[str, Any]]) -> None:
458
+ for item in feedback:
459
+ click.echo(
460
+ FeedbackManager.warning(message=f"△ {item.get('level')}: {item.get('resource')}: {item.get('message')}")
461
+ )
462
+
463
+
464
+ def format_build_errors(build_errors: list[dict[str, Any]]) -> str:
465
+ full_error_msg = ""
466
+ for build_error in build_errors:
467
+ filename_bit = build_error.get("filename", build_error.get("resource", ""))
468
+ error_bit = build_error.get("error") or build_error.get("message") or ""
469
+ error_msg = ((filename_bit + "\n") if filename_bit else "") + error_bit
470
+ full_error_msg += error_msg + "\n\n"
471
+ return full_error_msg.strip("\n") or "Unknown build error"
336
472
 
337
473
 
338
474
  def echo_changes(project: Project, changes: list[str], extension: str, status: str):
@@ -51,9 +51,9 @@ jobs:
51
51
  - name: Install Tinybird CLI
52
52
  run: curl https://tinybird.co | sh
53
53
  - name: Build project
54
- run: tb build
54
+ run: tb --local build
55
55
  - name: Test project
56
- run: tb test run
56
+ run: tb --local test run
57
57
  - name: Deployment check
58
58
  run: tb --cloud --host ${{! env.TINYBIRD_HOST }} --token ${{! env.TINYBIRD_TOKEN }} deploy --check
59
59
  """
@@ -115,8 +115,8 @@ tinybird_ci_workflow:
115
115
  script:
116
116
  - export PATH="$HOME/.local/bin:$PATH"
117
117
  - cd $CI_PROJECT_DIR/{{ data_project_dir }}
118
- - tb build
119
- - tb test run
118
+ - tb --local build
119
+ - tb --local test run
120
120
  - tb --cloud --host "$TINYBIRD_HOST" --token "$TINYBIRD_TOKEN" deploy --check
121
121
  services:
122
122
  - name: tinybirdco/tinybird-local:latest
@@ -625,8 +625,10 @@ def cli(
625
625
  config["dev_mode"] = tinybird_dev_mode
626
626
 
627
627
  # Resolve project folder from tinybird.config.json (preferred) or legacy .tinyb cwd.
628
- folder = get_project_folder_from_tinybird_config(os.getcwd())
629
- if not folder:
628
+ folder_from_config = get_project_folder_from_tinybird_config(os.getcwd())
629
+ if folder_from_config:
630
+ folder = folder_from_config
631
+ else:
630
632
  tinyb_dir = os.path.dirname(config_temp._path) # Directory containing .tinyb file
631
633
  cwd_config = config.get("cwd", ".")
632
634
 
@@ -870,9 +872,7 @@ def sql(
870
872
  if output == "json":
871
873
  echo_json(res, indent=8)
872
874
  else:
873
- dd = []
874
- for d in res["data"]:
875
- dd.append(d.values())
875
+ dd = [d.values() for d in res["data"]]
876
876
  echo_safe_format_table(dd, columns=res["meta"])
877
877
  else:
878
878
  click.echo(FeedbackManager.info_no_rows())