tinybird 0.0.1.dev68__tar.gz → 0.0.1.dev70__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (101) hide show
  1. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/PKG-INFO +11 -2
  2. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/ch_utils/engine.py +2 -4
  3. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/context.py +0 -1
  4. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/datafile.py +1 -1
  5. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/sql_template.py +1 -3
  6. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/sql_toolset.py +3 -3
  7. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/__cli__.py +2 -2
  8. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/auth.py +1 -1
  9. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/build.py +12 -1
  10. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/cli.py +15 -6
  11. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/common.py +9 -9
  12. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/build_common.py +1 -1
  13. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/build_datasource.py +1 -1
  14. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/common.py +18 -1
  15. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/fixture.py +7 -0
  16. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/pipe_checker.py +1 -1
  17. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datasource.py +65 -2
  18. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/deployment.py +31 -5
  19. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/endpoint.py +1 -1
  20. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/fmt.py +1 -1
  21. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/materialization.py +5 -5
  22. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/mock.py +26 -19
  23. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/pipe.py +1 -1
  24. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/project.py +3 -3
  25. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/shell.py +13 -21
  26. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/test.py +1 -1
  27. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/token.py +4 -4
  28. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/watch.py +1 -1
  29. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/workspace.py +6 -6
  30. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/workspace_members.py +6 -6
  31. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/common.py +2 -2
  32. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tornado_template.py +2 -1
  33. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird.egg-info/PKG-INFO +11 -2
  34. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/setup.cfg +0 -0
  35. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/__cli__.py +0 -0
  36. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/ch_utils/constants.py +0 -0
  37. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/check_pypi.py +0 -0
  38. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/client.py +0 -0
  39. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/config.py +0 -0
  40. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/connectors.py +0 -0
  41. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/datatypes.py +0 -0
  42. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/feedback_manager.py +0 -0
  43. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/git_settings.py +0 -0
  44. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/prompts.py +0 -0
  45. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/sql.py +0 -0
  46. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/sql_template_fmt.py +0 -0
  47. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/syncasync.py +0 -0
  48. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/cli.py +0 -0
  49. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/cicd.py +0 -0
  50. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/config.py +0 -0
  51. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/copy.py +0 -0
  52. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/create.py +0 -0
  53. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/build.py +0 -0
  54. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  55. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/diff.py +0 -0
  56. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  57. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/format_common.py +0 -0
  58. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  59. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  60. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
  61. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
  62. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/datafile/pull.py +0 -0
  63. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/exceptions.py +0 -0
  64. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/feedback_manager.py +0 -0
  65. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/job.py +0 -0
  66. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/llm.py +0 -0
  67. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/llm_utils.py +0 -0
  68. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/local.py +0 -0
  69. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/local_common.py +0 -0
  70. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/login.py +0 -0
  71. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/regions.py +0 -0
  72. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/table.py +0 -0
  73. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/tag.py +0 -0
  74. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/telemetry.py +0 -0
  75. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  76. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  77. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli.py +0 -0
  78. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/auth.py +0 -0
  79. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/branch.py +0 -0
  80. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/cicd.py +0 -0
  81. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/cli.py +0 -0
  82. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/config.py +0 -0
  83. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/connection.py +0 -0
  84. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/datasource.py +0 -0
  85. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/exceptions.py +0 -0
  86. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/fmt.py +0 -0
  87. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/job.py +0 -0
  88. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/pipe.py +0 -0
  89. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/regions.py +0 -0
  90. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/tag.py +0 -0
  91. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/telemetry.py +0 -0
  92. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/test.py +0 -0
  93. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  94. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  95. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/workspace.py +0 -0
  96. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  97. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird.egg-info/SOURCES.txt +0 -0
  98. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird.egg-info/dependency_links.txt +0 -0
  99. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird.egg-info/entry_points.txt +0 -0
  100. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird.egg-info/requires.txt +0 -0
  101. {tinybird-0.0.1.dev68 → tinybird-0.0.1.dev70}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev68
3
+ Version: 0.0.1.dev70
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -49,6 +49,15 @@ Requires-Dist: google-cloud-storage==2.4.0; extra == "snowflake"
49
49
  Requires-Dist: oauth2client==3.0.0; extra == "snowflake"
50
50
  Requires-Dist: chardet<4,>=3.0.2; extra == "snowflake"
51
51
  Requires-Dist: pyOpenSSL<20.0.0,>=16.2.0; extra == "snowflake"
52
+ Dynamic: author
53
+ Dynamic: author-email
54
+ Dynamic: description
55
+ Dynamic: description-content-type
56
+ Dynamic: home-page
57
+ Dynamic: provides-extra
58
+ Dynamic: requires-dist
59
+ Dynamic: requires-python
60
+ Dynamic: summary
52
61
 
53
62
  Tinybird CLI
54
63
  =============
@@ -456,7 +456,7 @@ ENABLED_ENGINES = [
456
456
  MERGETREE_OPTIONS,
457
457
  ),
458
458
  # AggregatingMergeTree()
459
- engine_config("AggregatingMergeTree", options=REPLACINGMERGETREE_OPTIONS),
459
+ engine_config("AggregatingMergeTree", options=MERGETREE_OPTIONS),
460
460
  # CollapsingMergeTree(sign)
461
461
  engine_config(
462
462
  "CollapsingMergeTree",
@@ -631,9 +631,7 @@ def engine_full_from_dict(
631
631
 
632
632
  >>> schema = 'sign_column Int8'
633
633
  >>> engine_full_from_dict('AggregatingMergeTree', {}, schema=schema)
634
- Traceback (most recent call last):
635
- ...
636
- ValueError: Missing required option 'sorting_key'
634
+ 'AggregatingMergeTree() ORDER BY (tuple())'
637
635
 
638
636
  >>> columns=[]
639
637
  >>> columns.append({'name': 'key_column', 'type': 'Int8', 'codec': None, 'default_value': None, 'nullable': False, 'normalized_name': 'key_column'})
@@ -20,5 +20,4 @@ engine: ContextVar[str] = ContextVar("engine")
20
20
  wait_parameter: ContextVar[bool] = ContextVar("wait_parameter")
21
21
  api_host: ContextVar[str] = ContextVar("api_host")
22
22
  ff_split_to_array_escape: ContextVar[bool] = ContextVar("ff_split_to_array_escape")
23
- ff_preprocess_parameters_circuit_breaker: ContextVar[bool] = ContextVar("ff_preprocess_parameters_circuit_breaker")
24
23
  ff_column_json_backticks_circuit_breaker: ContextVar[bool] = ContextVar("ff_column_json_backticks_circuit_breaker")
@@ -2301,7 +2301,7 @@ class PipeCheckerRunner:
2301
2301
  )
2302
2302
 
2303
2303
  result = PipeCheckerTextTestResult(
2304
- self.checker_stream_result_class(sys.stdout), # type: ignore
2304
+ self.checker_stream_result_class(sys.stdout),
2305
2305
  descriptions=True,
2306
2306
  verbosity=2,
2307
2307
  custom_output=custom_output,
@@ -14,7 +14,6 @@ from tornado.util import ObjectDict, exec_in, unicode_type
14
14
 
15
15
  from tinybird.context import (
16
16
  ff_column_json_backticks_circuit_breaker,
17
- ff_preprocess_parameters_circuit_breaker,
18
17
  ff_split_to_array_escape,
19
18
  )
20
19
 
@@ -2244,14 +2243,13 @@ def render_sql_template(
2244
2243
  tinybird.sql_template.SQLTemplateException: Template Syntax Error: Required parameter is not defined. Check the parameters test. Please provide a value or set a default value in the pipe code.
2245
2244
  """
2246
2245
  escape_split_to_array = ff_split_to_array_escape.get(False)
2247
- bypass_preprocess_variables = ff_preprocess_parameters_circuit_breaker.get(False)
2248
2246
 
2249
2247
  t, template_variables, variable_warnings = get_template_and_variables(
2250
2248
  sql, name, escape_arrays=escape_split_to_array
2251
2249
  )
2252
2250
  template_variables_with_types = get_var_names_and_types_cached(t)
2253
2251
 
2254
- if not bypass_preprocess_variables and variables is not None:
2252
+ if variables is not None:
2255
2253
  processed_variables = preprocess_variables(variables, template_variables_with_types)
2256
2254
  variables.update(processed_variables)
2257
2255
 
@@ -3,7 +3,7 @@ import logging
3
3
  from collections import defaultdict
4
4
  from datetime import datetime
5
5
  from functools import lru_cache
6
- from typing import FrozenSet, List, Optional, Set, Tuple
6
+ from typing import FrozenSet, List, Optional, Set, Tuple, Union
7
7
 
8
8
  from chtoolset import query as chquery
9
9
  from toposort import toposort
@@ -172,7 +172,7 @@ def tables_or_sql(replacement: dict, table_functions=False) -> set:
172
172
  return {replacement}
173
173
 
174
174
 
175
- def _separate_as_tuple_if_contains_database_and_table(definition: str) -> str | Tuple[str, str]:
175
+ def _separate_as_tuple_if_contains_database_and_table(definition: str) -> Union[str, Tuple[str, str]]:
176
176
  if "." in definition:
177
177
  database_and_table_separated = definition.split(".")
178
178
  return database_and_table_separated[0], database_and_table_separated[1]
@@ -255,7 +255,7 @@ def replace_tables(
255
255
  function_allow_list=function_allow_list,
256
256
  )
257
257
  seen_tables = set()
258
- table: Tuple[str, str] | Tuple[str, str, str]
258
+ table: Union[Tuple[str, str], Tuple[str, str, str]]
259
259
  if function_allow_list is None:
260
260
  _enabled_table_functions = ENABLED_TABLE_FUNCTIONS
261
261
  else:
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev68'
8
- __revision__ = 'e28d545'
7
+ __version__ = '0.0.1.dev70'
8
+ __revision__ = 'f8064d4'
@@ -30,7 +30,7 @@ from tinybird.tb.modules.regions import Region
30
30
  @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
31
31
  @click.option(
32
32
  "--host",
33
- help="Set custom host if it's different than https://api.tinybird.co. Check https://www.tinybird.co/docs/api-reference/overview#regions-and-endpoints for the available list of regions",
33
+ help="Set custom host if it's different than https://api.tinybird.co. Check https://www.tinybird.co/docs/api-reference#regions-and-endpoints for the available list of regions",
34
34
  )
35
35
  @click.option(
36
36
  "--region", envvar="TB_REGION", help="Set region. Run 'tb auth ls' to show available regions. Overrides host."
@@ -15,7 +15,7 @@ from tinybird.client import TinyB
15
15
  from tinybird.tb.modules.cli import cli
16
16
  from tinybird.tb.modules.common import push_data
17
17
  from tinybird.tb.modules.datafile.build import folder_build
18
- from tinybird.tb.modules.datafile.fixture import get_fixture_dir
18
+ from tinybird.tb.modules.datafile.fixture import get_fixture_dir, persist_fixture
19
19
  from tinybird.tb.modules.feedback_manager import FeedbackManager
20
20
  from tinybird.tb.modules.project import Project
21
21
  from tinybird.tb.modules.shell import Shell, print_table_formatted
@@ -210,6 +210,8 @@ def process(
210
210
  build_failed = False
211
211
  if file_changed and file_changed.endswith(".ndjson"):
212
212
  rebuild_fixture(project, tb_client, file_changed)
213
+ elif file_changed and file_changed.endswith(".sql"):
214
+ rebuild_fixture_sql(project, tb_client, file_changed)
213
215
  else:
214
216
  try:
215
217
  build_project(project, tb_client, file_changed)
@@ -247,3 +249,12 @@ def run_watch(
247
249
  )
248
250
  watcher_thread.start()
249
251
  shell.run()
252
+
253
+
254
+ def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> None:
255
+ sql_path = Path(sql_file)
256
+ datasource_name = sql_path.stem
257
+ sql = sql_path.read_text()
258
+ result = asyncio.run(tb_client.query(f"{sql} FORMAT JSON"))
259
+ data = result.get("data", [])
260
+ persist_fixture(datasource_name, data, project.folder)
@@ -127,6 +127,7 @@ async def cli(
127
127
  ctx.ensure_object(dict)["client"] = client
128
128
 
129
129
  ctx.ensure_object(dict)["project"] = project
130
+ ctx.ensure_object(dict)["env"] = get_target_env(cloud, build)
130
131
 
131
132
 
132
133
  @cli.command(hidden=True)
@@ -147,12 +148,12 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
147
148
  @click.option("--no-deps", is_flag=True, default=False, help="Print only data sources with no pipes using them")
148
149
  @click.option("--match", default=None, help="Retrieve any resource matching the pattern")
149
150
  @click.option("--pipe", default=None, help="Retrieve any resource used by pipe")
150
- @click.option("--datasource", default=None, help="Retrieve resources depending on this Data Source")
151
+ @click.option("--datasource", default=None, help="Retrieve resources depending on this data source")
151
152
  @click.option(
152
153
  "--check-for-partial-replace",
153
154
  is_flag=True,
154
155
  default=False,
155
- help="Retrieve dependant Data Sources that will have their data replaced if a partial replace is executed in the Data Source selected",
156
+ help="Retrieve dependant data sources that will have their data replaced if a partial replace is executed in the data source selected",
156
157
  )
157
158
  @click.option("--recursive", is_flag=True, default=False, help="Calculate recursive dependencies")
158
159
  @click.pass_context
@@ -186,7 +187,7 @@ async def dependencies(
186
187
 
187
188
  @cli.command(
188
189
  name="diff",
189
- short_help="Diffs local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
190
+ short_help="Diff local datafiles to the corresponding remote files in the workspace. For the case of .datasource files it just diffs VERSION and SCHEMA, since ENGINE, KAFKA or other metadata is considered immutable.",
190
191
  )
191
192
  @click.argument("filename", type=click.Path(exists=True), nargs=-1, required=False)
192
193
  @click.option(
@@ -203,7 +204,7 @@ async def dependencies(
203
204
  "--main",
204
205
  is_flag=True,
205
206
  default=False,
206
- help="Diffs local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on a Branch.",
207
+ help="Diff local datafiles to the corresponding remote files in the main workspace. Only works when authenticated on a Branch.",
207
208
  hidden=True,
208
209
  )
209
210
  @click.pass_context
@@ -255,7 +256,7 @@ async def diff(
255
256
  @cli.command()
256
257
  @click.argument("query", required=False)
257
258
  @click.option("--rows_limit", default=100, help="Max number of rows retrieved")
258
- @click.option("--pipeline", default=None, help="The name of the Pipe to run the SQL Query")
259
+ @click.option("--pipeline", default=None, help="The name of the pipe to run the SQL Query")
259
260
  @click.option("--pipe", default=None, help="The path to the .pipe file to run the SQL Query of a specific NODE")
260
261
  @click.option("--node", default=None, help="The NODE name")
261
262
  @click.option(
@@ -393,7 +394,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
393
394
 
394
395
  commands_always_cloud = ["pull"]
395
396
  commands_always_build = ["build", "test", "dev"]
396
- commands_always_local = ["create", "mock"]
397
+ commands_always_local = ["create"]
397
398
  if (
398
399
  (cloud or command in commands_always_cloud)
399
400
  and command not in commands_always_build
@@ -407,3 +408,11 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, b
407
408
  if not build and command not in commands_always_local and command not in commands_always_build:
408
409
  click.echo(FeedbackManager.gray(message="Running against Tinybird Local\n"))
409
410
  return await get_tinybird_local_client(config, build=build)
411
+
412
+
413
+ def get_target_env(cloud: bool, build: bool) -> str:
414
+ if cloud:
415
+ return "cloud"
416
+ if build:
417
+ return "build"
418
+ return "local"
@@ -571,7 +571,7 @@ def region_from_host(region_name_or_host, regions):
571
571
 
572
572
  def ask_for_user_token(action: str, ui_host: str) -> str:
573
573
  return click.prompt(
574
- f'\nUse the token called "user token" in order to {action}. Copy it from {ui_host}/tokens and paste it here',
574
+ f'\nUse the token called "user token" to {action}. Copy it from {ui_host}/tokens and paste it here',
575
575
  hide_input=True,
576
576
  show_default=False,
577
577
  default=None,
@@ -591,13 +591,13 @@ async def check_user_token(ctx: Context, token: str):
591
591
  if not is_authenticated.get("is_valid", False):
592
592
  raise CLIWorkspaceException(
593
593
  FeedbackManager.error_exception(
594
- error='Invalid token. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
594
+ error='Invalid token. Make sure you are using the "user token" instead of the "admin your@email" token.'
595
595
  )
596
596
  )
597
597
  if is_authenticated.get("is_valid") and not is_authenticated.get("is_user", False):
598
598
  raise CLIWorkspaceException(
599
599
  FeedbackManager.error_exception(
600
- error='Invalid user authentication. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
600
+ error='Invalid user authentication. Make sure you are using the "user token" instead of the "admin your@email" token.'
601
601
  )
602
602
  )
603
603
 
@@ -614,13 +614,13 @@ async def check_user_token_with_client(client: TinyB, token: str):
614
614
  if not is_authenticated.get("is_valid", False):
615
615
  raise CLIWorkspaceException(
616
616
  FeedbackManager.error_exception(
617
- error='Invalid token. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
617
+ error='Invalid token. Make sure you are using the "user token" instead of the "admin your@email" token.'
618
618
  )
619
619
  )
620
620
  if is_authenticated.get("is_valid") and not is_authenticated.get("is_user", False):
621
621
  raise CLIWorkspaceException(
622
622
  FeedbackManager.error_exception(
623
- error='Invalid user authentication. Please, be sure you are using the "user token" instead of the "admin your@email" token.'
623
+ error='Invalid user authentication. Make sure you are using the "user token" instead of the "admin your@email" token.'
624
624
  )
625
625
  )
626
626
 
@@ -1153,17 +1153,17 @@ def validate_kafka_bootstrap_servers(host_and_port):
1153
1153
 
1154
1154
  def validate_kafka_key(s):
1155
1155
  if not isinstance(s, str):
1156
- raise CLIException("Key format is not correct, it should be a string")
1156
+ raise CLIException("Key format is not correct, it must be a string")
1157
1157
 
1158
1158
 
1159
1159
  def validate_kafka_secret(s):
1160
1160
  if not isinstance(s, str):
1161
- raise CLIException("Password format is not correct, it should be a string")
1161
+ raise CLIException("Password format is not correct, it must be a string")
1162
1162
 
1163
1163
 
1164
1164
  def validate_string_connector_param(param, s):
1165
1165
  if not isinstance(s, str):
1166
- raise CLIConnectionException(param + " format is not correct, it should be a string")
1166
+ raise CLIConnectionException(param + " format is not correct, it must be a string")
1167
1167
 
1168
1168
 
1169
1169
  async def validate_connection_name(client, connection_name, service):
@@ -1434,7 +1434,7 @@ async def try_update_config_with_remote(
1434
1434
  def ask_for_admin_token_interactively(ui_host: str, default_token: Optional[str]) -> str:
1435
1435
  return (
1436
1436
  click.prompt(
1437
- f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from .tinyb file" if default_token else ""}',
1437
+ f'\nCopy the "admin your@email" token from {ui_host}/tokens and paste it here {"OR press enter to use the token from the .tinyb file" if default_token else ""}',
1438
1438
  hide_input=True,
1439
1439
  show_default=False,
1440
1440
  default=default_token,
@@ -94,7 +94,7 @@ async def update_tags_in_resource(rs: Dict[str, Any], resource_type: str, client
94
94
  resource_name = persisted_ds.get("name", "")
95
95
  except DoesNotExistException:
96
96
  click.echo(
97
- FeedbackManager.error_tag_generic("Could not get the latest Data Source info for updating its tags.")
97
+ FeedbackManager.error_tag_generic("Could not get the latest data source info for updating its tags.")
98
98
  )
99
99
  elif resource_type == "pipe":
100
100
  pipe_name = rs["name"]
@@ -60,7 +60,7 @@ async def new_ds(
60
60
 
61
61
  if engine_param.lower() == "join":
62
62
  deprecation_notice = FeedbackManager.warning_deprecated(
63
- warning="Data Sources with Join engine are deprecated and will be removed in the next major release of tinybird-cli. Use MergeTree instead."
63
+ warning="Data sources with Join engine are deprecated and will be removed in the next major release of tinybird-cli. Use MergeTree instead."
64
64
  )
65
65
  click.echo(deprecation_notice)
66
66
 
@@ -20,6 +20,7 @@ from string import Template
20
20
  from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, cast
21
21
 
22
22
  import click
23
+ from croniter import croniter
23
24
  from mypy_extensions import KwArg, VarArg
24
25
 
25
26
  from tinybird.ch_utils.engine import ENABLED_ENGINES
@@ -200,6 +201,20 @@ class Datafile:
200
201
  def set_kind(self, kind: DatafileKind):
201
202
  self.kind = kind
202
203
 
204
+ def validate_copy_node(self, node: Dict[str, Any]):
205
+ if "target_datasource" not in node:
206
+ raise DatafileValidationError("COPY node missing target datasource")
207
+ # copy mode must be append or replace
208
+ if node.get("mode") and node["mode"] not in ["append", "replace"]:
209
+ raise DatafileValidationError("COPY node mode must be append or replace")
210
+ # copy schedule must be @on-demand or a cron-expression
211
+ if (
212
+ node.get("copy_schedule")
213
+ and node["copy_schedule"] != ON_DEMAND
214
+ and not croniter.is_valid(node["copy_schedule"])
215
+ ):
216
+ raise DatafileValidationError("COPY node schedule must be @on-demand or a valid cron expression.")
217
+
203
218
  def validate(self):
204
219
  if self.kind == DatafileKind.pipe:
205
220
  # TODO(eclbg):
@@ -208,7 +223,7 @@ class Datafile:
208
223
  # [x] Materialized nodes have target datasource
209
224
  # [x] Only one materialized node
210
225
  # [x] Only one node of any specific type
211
- # [ ] ...
226
+ # (rbarbadillo): there's a HUGE amount of validations in api_pipes.py, we should somehow merge them
212
227
  for node in self.nodes:
213
228
  if "sql" not in node:
214
229
  raise DatafileValidationError(f"SQL missing for node {repr(node['name'])}")
@@ -220,6 +235,8 @@ class Datafile:
220
235
  raise DatafileValidationError("Multiple non-standard nodes in pipe. There can only be one")
221
236
  if node.get("type", "").lower() == PipeNodeTypes.MATERIALIZED and "datasource" not in node:
222
237
  raise DatafileValidationError(f"Materialized node {repr(node['name'])} missing target datasource")
238
+ if node.get("type", "").lower() == PipeNodeTypes.COPY:
239
+ self.validate_copy_node(node)
223
240
  elif self.kind == DatafileKind.datasource:
224
241
  # TODO(eclbg):
225
242
  # [x] Just one node
@@ -8,6 +8,13 @@ def get_fixture_dir(folder: str) -> Path:
8
8
  return Path(folder) / "fixtures"
9
9
 
10
10
 
11
+ def persist_fixture_sql(fixture_name: str, sql: str, folder: str) -> Path:
12
+ fixture_dir = get_fixture_dir(folder)
13
+ fixture_file = fixture_dir / f"{fixture_name}.sql"
14
+ fixture_file.write_text(sql)
15
+ return fixture_file
16
+
17
+
11
18
  def persist_fixture(fixture_name: str, data: Union[List[Dict[str, Any]], str], folder: str, format="ndjson") -> Path:
12
19
  fixture_dir = get_fixture_dir(folder)
13
20
  fixture_file = fixture_dir / f"{fixture_name}.{format}"
@@ -412,7 +412,7 @@ class PipeCheckerRunner:
412
412
  )
413
413
 
414
414
  result = PipeCheckerTextTestResult(
415
- self.checker_stream_result_class(sys.stdout), # type: ignore
415
+ self.checker_stream_result_class(sys.stdout),
416
416
  descriptions=True,
417
417
  verbosity=2,
418
418
  custom_output=custom_output,
@@ -5,6 +5,7 @@
5
5
 
6
6
  import asyncio
7
7
  import json
8
+ import os
8
9
  import re
9
10
  from typing import Optional
10
11
 
@@ -23,8 +24,10 @@ from tinybird.tb.modules.common import (
23
24
  push_data,
24
25
  )
25
26
  from tinybird.tb.modules.datafile.common import get_name_version
27
+ from tinybird.tb.modules.datafile.fixture import persist_fixture
26
28
  from tinybird.tb.modules.exceptions import CLIDatasourceException
27
29
  from tinybird.tb.modules.feedback_manager import FeedbackManager
30
+ from tinybird.tb.modules.project import Project
28
31
 
29
32
 
30
33
  @cli.group()
@@ -34,7 +37,7 @@ def datasource(ctx):
34
37
 
35
38
 
36
39
  @datasource.command(name="ls")
37
- @click.option("--match", default=None, help="Retrieve any resources matching the pattern. eg --match _test")
40
+ @click.option("--match", default=None, help="Retrieve any resources matching the pattern. For example, --match _test")
38
41
  @click.option(
39
42
  "--format",
40
43
  "format_",
@@ -116,7 +119,7 @@ async def datasource_append(
116
119
  concurrency: int,
117
120
  ):
118
121
  """
119
- Appends data to an existing Data Source from URL, local file or a connector
122
+ Appends data to an existing data source from URL, local file or a connector
120
123
 
121
124
  - Load from URL `tb datasource append [datasource_name] https://url_to_csv`
122
125
 
@@ -391,3 +394,63 @@ async def datasource_data(ctx: Context, datasource: str, limit: int):
391
394
  echo_safe_humanfriendly_tables_format_smart_table(
392
395
  data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
393
396
  )
397
+
398
+
399
+ @datasource.command(name="export")
400
+ @click.argument("datasource")
401
+ @click.option(
402
+ "--format",
403
+ "format_",
404
+ type=click.Choice(["csv", "ndjson"], case_sensitive=False),
405
+ default="ndjson",
406
+ help="Output format (csv or ndjson)",
407
+ )
408
+ @click.option("--rows", type=int, default=100, help="Number of rows to export (default: 100)")
409
+ @click.option("--where", type=str, default=None, help="Condition to filter data")
410
+ @click.option("--target", type=str, help="Target file path (default: datasource_name.{format})")
411
+ @click.pass_context
412
+ @coro
413
+ async def datasource_export(
414
+ ctx: Context,
415
+ datasource: str,
416
+ format_: str,
417
+ rows: int,
418
+ where: Optional[str],
419
+ target: Optional[str],
420
+ ):
421
+ """Export data from a datasource to a file in CSV or NDJSON format
422
+
423
+ Example usage:
424
+ - Export all rows as CSV: tb datasource export my_datasource
425
+ - Export 1000 rows as NDJSON: tb datasource export my_datasource --format ndjson --rows 1000
426
+ - Export to specific file: tb datasource export my_datasource --output ./data/export.csv
427
+ """
428
+ client: TinyB = ctx.ensure_object(dict)["client"]
429
+ project: Project = ctx.ensure_object(dict)["project"]
430
+
431
+ # Determine output filename if not provided
432
+ if not target:
433
+ target = f"{datasource}.{format_}"
434
+
435
+ # Build query with optional row limit
436
+ query = f"SELECT * FROM {datasource} WHERE {where or 1} LIMIT {rows}"
437
+
438
+ click.echo(FeedbackManager.highlight(message=f"\n» Exporting {datasource} to {target}"))
439
+
440
+ try:
441
+ if format_ == "csv":
442
+ query += " FORMAT CSVWithNames"
443
+ else:
444
+ query += " FORMAT JSONEachRow"
445
+
446
+ res = await client.query(query)
447
+
448
+ fixture_path = persist_fixture(datasource, res, project.folder)
449
+ file_size = os.path.getsize(fixture_path)
450
+
451
+ click.echo(
452
+ FeedbackManager.success(message=f"✓ Exported data to {target} ({humanfriendly.format_size(file_size)})")
453
+ )
454
+
455
+ except Exception as e:
456
+ raise CLIDatasourceException(FeedbackManager.error(message=str(e)))
@@ -44,12 +44,15 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
44
44
  if candidate_deployment.get("live"):
45
45
  click.echo(FeedbackManager.error(message="Candidate deployment is already live"))
46
46
  else:
47
- click.echo(FeedbackManager.success(message="Promoting deployment"))
47
+ click.echo(FeedbackManager.success(message="Setting candidate deployment as live"))
48
48
 
49
49
  TINYBIRD_API_URL = f"{host}/v1/deployments/{candidate_deployment.get('id')}/set-live"
50
50
  r = requests.post(TINYBIRD_API_URL, headers=headers)
51
51
  result = r.json()
52
52
  logging.debug(json.dumps(result, indent=2))
53
+ if result.get("error"):
54
+ click.echo(FeedbackManager.error(message=result.get("error")))
55
+ sys.exit(1)
53
56
 
54
57
  click.echo(FeedbackManager.success(message="Removing old deployment"))
55
58
 
@@ -57,6 +60,9 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
57
60
  r = requests.delete(TINYBIRD_API_URL, headers=headers)
58
61
  result = r.json()
59
62
  logging.debug(json.dumps(result, indent=2))
63
+ if result.get("error"):
64
+ click.echo(FeedbackManager.error(message=result.get("error")))
65
+ sys.exit(1)
60
66
 
61
67
  click.echo(FeedbackManager.success(message="Deployment promotion successfully started"))
62
68
 
@@ -110,6 +116,9 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
110
116
  r = requests.post(TINYBIRD_API_URL, headers=headers)
111
117
  result = r.json()
112
118
  logging.debug(json.dumps(result, indent=2))
119
+ if result.get("error"):
120
+ click.echo(FeedbackManager.error(message=result.get("error")))
121
+ sys.exit(1)
113
122
 
114
123
  click.echo(FeedbackManager.success(message="Removing current deployment"))
115
124
 
@@ -117,6 +126,9 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
117
126
  r = requests.delete(TINYBIRD_API_URL, headers=headers)
118
127
  result = r.json()
119
128
  logging.debug(json.dumps(result, indent=2))
129
+ if result.get("error"):
130
+ click.echo(FeedbackManager.error(message=result.get("error")))
131
+ sys.exit(1)
120
132
 
121
133
  click.echo(FeedbackManager.success(message="Deployment rollback successfully started"))
122
134
 
@@ -254,13 +266,13 @@ def deployment_rollback(ctx: click.Context, wait: bool) -> None:
254
266
  @click.option(
255
267
  "--wait/--no-wait",
256
268
  is_flag=True,
257
- default=False,
269
+ default=True,
258
270
  help="Wait for deploy to finish. Disabled by default.",
259
271
  )
260
272
  @click.option(
261
273
  "--auto/--no-auto",
262
274
  is_flag=True,
263
- default=False,
275
+ default=True,
264
276
  help="Auto-promote the deployment. Only works if --wait is enabled. Disabled by default.",
265
277
  )
266
278
  @click.option(
@@ -299,6 +311,7 @@ def create_deployment(
299
311
  }
300
312
  project: Project = ctx.ensure_object(dict)["project"]
301
313
  client = ctx.ensure_object(dict)["client"]
314
+ config = ctx.ensure_object(dict)["config"]
302
315
  TINYBIRD_API_URL = f"{client.host}/v1/deploy"
303
316
  TINYBIRD_API_KEY = client.token
304
317
 
@@ -345,8 +358,21 @@ def create_deployment(
345
358
  deploy_result = result.get("result")
346
359
  if deploy_result == "success":
347
360
  print_changes(result, project)
348
- click.echo(FeedbackManager.success(message="Deployment submitted successfully"))
349
- deployment = result.get("deployment")
361
+ deployment = result.get("deployment", {})
362
+ # We show the url in the case of region is public
363
+ if client.host == "https://api.europe-west2.gcp.tinybird.co":
364
+ click.echo(
365
+ FeedbackManager.gray(message="Deployment URL: ")
366
+ + FeedbackManager.info(
367
+ message=f"https://cloud.tinybird.co/gcp/europe-west2/{config.get('name')}/deployments/{deployment.get('id')}"
368
+ )
369
+ )
370
+
371
+ if wait:
372
+ click.echo(FeedbackManager.info(message="\n✓ Deployment submitted successfully"))
373
+ else:
374
+ click.echo(FeedbackManager.success(message="\n✓ Deployment submitted successfully"))
375
+
350
376
  feedback = deployment.get("feedback", [])
351
377
  for f in feedback:
352
378
  click.echo(
@@ -29,7 +29,7 @@ def endpoint(ctx):
29
29
 
30
30
 
31
31
  @endpoint.command(name="ls")
32
- @click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
32
+ @click.option("--match", default=None, help="Retrieve any resource matching the pattern. For example, --match _test")
33
33
  @click.option(
34
34
  "--format",
35
35
  "format_",
@@ -58,7 +58,7 @@ async def fmt(
58
58
  elif (".pipe" in extensions) or (".incl" in extensions):
59
59
  result = await format_pipe(filename, line_length, skip_eval=True)
60
60
  else:
61
- click.echo("Unsupported file type. Supported files types are: .pipe, .incl and .datasource")
61
+ click.echo("Unsupported file type. Supported files types are: .pipe and .datasource")
62
62
  return None
63
63
 
64
64
  if diff:
@@ -23,7 +23,7 @@ def materialization(ctx):
23
23
 
24
24
 
25
25
  @materialization.command(name="ls")
26
- @click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
26
+ @click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. For example, --match _test")
27
27
  @click.option(
28
28
  "--format",
29
29
  "format_",
@@ -79,16 +79,16 @@ async def materialization_ls(ctx: click.Context, match: str, format_: str):
79
79
  "--sql-condition",
80
80
  type=str,
81
81
  default=None,
82
- help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
82
+ help="Populate with a SQL condition to be applied to the trigger data source of the materialized view. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger data source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the data source ``engine_sorting_key`` will make the populate job process less data.",
83
83
  )
84
84
  @click.option(
85
- "--truncate", is_flag=True, default=False, help="Truncates the materialized Data Source before populating it."
85
+ "--truncate", is_flag=True, default=False, help="Truncates the materialized data source before populating it."
86
86
  )
87
87
  @click.option(
88
88
  "--unlink-on-populate-error",
89
89
  is_flag=True,
90
90
  default=False,
91
- help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
91
+ help="If the populate job fails the materialized view is unlinked and new data won't be ingested in the materialized view. First time a populate job fails, the materialized view is always unlinked.",
92
92
  )
93
93
  @click.option(
94
94
  "--wait",
@@ -107,7 +107,7 @@ async def pipe_populate(
107
107
  unlink_on_populate_error: bool,
108
108
  wait: bool,
109
109
  ):
110
- """Populate the result of a Materialized Node into the target Materialized View"""
110
+ """Populate the result of a Materialized Node into the target materialized view"""
111
111
  cl = create_tb_client(ctx)
112
112
 
113
113
  pipe = await cl.pipe(pipe_name)