tinybird 4.6.4__tar.gz → 4.6.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. {tinybird-4.6.4 → tinybird-4.6.5}/PKG-INFO +6 -1
  2. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/ch_utils/engine.py +0 -25
  3. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/client.py +19 -3
  4. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/config.py +0 -6
  5. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/datafile/common.py +0 -15
  6. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/datatypes.py +0 -4
  7. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/feedback_manager.py +5 -0
  8. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/prompts.py +2 -272
  9. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/service_datasources.py +1 -18
  10. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/sql.py +0 -24
  11. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/sql_template.py +0 -29
  12. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/__cli__.py +2 -2
  13. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/client.py +30 -2
  14. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/config.py +0 -6
  15. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/build.py +1 -59
  16. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/cli.py +0 -4
  17. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/common.py +0 -74
  18. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/config.py +0 -8
  19. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/connection_s3.py +4 -8
  20. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/build.py +5 -249
  21. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/diff.py +2 -2
  22. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/fixture.py +0 -19
  23. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/playground.py +4 -4
  24. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/pull.py +9 -1
  25. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/llm_utils.py +0 -8
  26. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/local_common.py +0 -1
  27. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/local_logs.py +0 -4
  28. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/login_common.py +3 -2
  29. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/test_common.py +0 -5
  30. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -9
  31. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/branch.py +0 -1
  32. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/common.py +0 -16
  33. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/config.py +0 -8
  34. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/datasource.py +73 -4
  35. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -9
  36. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/workspace.py +0 -1
  37. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird.egg-info/PKG-INFO +6 -1
  38. {tinybird-4.6.4 → tinybird-4.6.5}/setup.cfg +0 -0
  39. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/__cli__.py +0 -0
  40. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/ch_utils/constants.py +0 -0
  41. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/check_pypi.py +0 -0
  42. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/context.py +0 -0
  43. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/datafile/exceptions.py +0 -0
  44. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/datafile/parse_connection.py +0 -0
  45. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/datafile/parse_datasource.py +0 -0
  46. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/datafile/parse_pipe.py +0 -0
  47. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/git_settings.py +0 -0
  48. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/iterating/__init__.py +0 -0
  49. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/iterating/data_branch_modes.py +0 -0
  50. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/sql_template_fmt.py +0 -0
  51. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/sql_toolset.py +0 -0
  52. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/syncasync.py +0 -0
  53. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/check_pypi.py +0 -0
  54. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/cli.py +0 -0
  55. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/branch.py +0 -0
  56. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/build_common.py +0 -0
  57. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/cicd.py +0 -0
  58. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/connection.py +0 -0
  59. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/connection_dynamodb.py +0 -0
  60. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/connection_kafka.py +0 -0
  61. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/copy.py +0 -0
  62. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/create.py +0 -0
  63. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/build_common.py +0 -0
  64. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  65. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  66. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/format_common.py +0 -0
  67. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/format_connection.py +0 -0
  68. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  69. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  70. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  71. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/datasource.py +0 -0
  72. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/deployment.py +0 -0
  73. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/deployment_common.py +0 -0
  74. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/deprecations.py +0 -0
  75. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/endpoint.py +0 -0
  76. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/exceptions.py +0 -0
  77. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/feedback_manager.py +0 -0
  78. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/fmt.py +0 -0
  79. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/info.py +0 -0
  80. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/infra.py +0 -0
  81. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/job.py +0 -0
  82. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/job_common.py +0 -0
  83. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/llm.py +0 -0
  84. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/local.py +0 -0
  85. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/login.py +0 -0
  86. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/logout.py +0 -0
  87. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/logs.py +0 -0
  88. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/materialization.py +0 -0
  89. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/open.py +0 -0
  90. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/pipe.py +0 -0
  91. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/preview.py +0 -0
  92. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/project.py +0 -0
  93. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/project_commands.py +0 -0
  94. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/py_project.py +0 -0
  95. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/query_output.py +0 -0
  96. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/regions.py +0 -0
  97. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/secret.py +0 -0
  98. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/secret_common.py +0 -0
  99. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/sink.py +0 -0
  100. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/table.py +0 -0
  101. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/telemetry.py +0 -0
  102. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/test.py +0 -0
  103. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/token.py +0 -0
  104. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/ts_project.py +0 -0
  105. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/watch.py +0 -0
  106. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/workspace.py +0 -0
  107. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb/modules/workspace_members.py +0 -0
  108. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli.py +0 -0
  109. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/auth.py +0 -0
  110. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/cicd.py +0 -0
  111. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/cli.py +0 -0
  112. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/connection.py +0 -0
  113. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/exceptions.py +0 -0
  114. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/fmt.py +0 -0
  115. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/job.py +0 -0
  116. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/pipe.py +0 -0
  117. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/regions.py +0 -0
  118. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/tag.py +0 -0
  119. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/telemetry.py +0 -0
  120. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/test.py +0 -0
  121. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  122. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird/tornado_template.py +0 -0
  123. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird.egg-info/SOURCES.txt +0 -0
  124. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird.egg-info/dependency_links.txt +0 -0
  125. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird.egg-info/entry_points.txt +0 -0
  126. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird.egg-info/requires.txt +0 -0
  127. {tinybird-4.6.4 → tinybird-4.6.5}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 4.6.4
3
+ Version: 4.6.5
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -52,6 +52,11 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
52
52
  Changelog
53
53
  ----------
54
54
 
55
+ 4.6.5
56
+ *******
57
+
58
+ - `Changed` `tb pull` now automatically creates cloud secrets and shows full Kafka connections.
59
+
55
60
  4.6.4
56
61
  *******
57
62
 
@@ -354,31 +354,6 @@ def sorting_key_is_valid(columns: List[Dict[str, Any]], value: Optional[str]) ->
354
354
  return value
355
355
 
356
356
 
357
- def case_insensitive_check(valid_values: List[str]) -> Callable[[List[Dict[str, Any]], str], Optional[str]]:
358
- """
359
- >>> valid_values = ['ANY', 'ALL']
360
- >>> checker = case_insensitive_check(valid_values)
361
- >>> checker([],'ALL')
362
-
363
- >>> valid_values = ['ANY', 'ALL']
364
- >>> checker = case_insensitive_check(valid_values)
365
- >>> checker([],'any')
366
-
367
- >>> valid_values = ['ANY', 'ALL']
368
- >>> checker = case_insensitive_check(valid_values)
369
- >>> checker([],'foo')
370
- Traceback (most recent call last):
371
- ...
372
- ValueError: valid values are ANY, ALL
373
- """
374
-
375
- def checker(columns: List[Dict[str, Any]], value: str):
376
- if value.upper() not in valid_values:
377
- raise ValueError(f"valid values are {', '.join(valid_values)}")
378
-
379
- return checker
380
-
381
-
382
357
  # [PARTITION BY expr]
383
358
  # [ORDER BY expr]
384
359
  # [PRIMARY KEY expr]
@@ -470,10 +470,26 @@ class TinyB:
470
470
  async def datasource_truncate(self, datasource_name: str):
471
471
  return await self._req(f"/v0/datasources/{datasource_name}/truncate", method="POST", data="")
472
472
 
473
- async def datasource_delete_rows(self, datasource_name: str, delete_condition: str, dry_run: bool = False):
474
- params = {"delete_condition": delete_condition}
473
+ async def datasource_delete_rows(
474
+ self,
475
+ datasource_name: str,
476
+ delete_condition: str,
477
+ dry_run: bool = False,
478
+ lightweight: bool = False,
479
+ wait: bool = True,
480
+ partition: Optional[str] = None,
481
+ projection_mode: Optional[str] = None,
482
+ ):
483
+ params: Dict[str, Any] = {"delete_condition": delete_condition}
484
+ if lightweight:
485
+ params["wait"] = "true" if wait else "false"
486
+ if partition:
487
+ params["partition"] = partition
488
+ if projection_mode:
489
+ params["projection_mode"] = projection_mode
490
+ return await self._req(f"/v1/datasources/{datasource_name}/delete", method="POST", data=params)
475
491
  if dry_run:
476
- params.update({"dry_run": "true"})
492
+ params["dry_run"] = "true"
477
493
  return await self._req(f"/v0/datasources/{datasource_name}/delete", method="POST", data=params)
478
494
 
479
495
  async def datasource_dependencies(
@@ -112,12 +112,6 @@ async def get_config(
112
112
  return config
113
113
 
114
114
 
115
- async def write_config(config: Dict[str, Any], dest_file: str = ".tinyb"):
116
- config_file = Path(getcwd()) / dest_file
117
- async with aiofiles.open(config_file, "w") as file:
118
- await file.write(json.dumps(config, indent=4, sort_keys=True))
119
-
120
-
121
115
  def get_display_host(ui_host: str):
122
116
  return LEGACY_HOSTS.get(ui_host, ui_host)
123
117
 
@@ -96,10 +96,6 @@ class IndexesSyntaxError(DatafileSyntaxError):
96
96
  super().__init__(message=message, lineno=lineno, pos=pos, hint=hint)
97
97
 
98
98
 
99
- class MalformedColumnError(Exception):
100
- pass
101
-
102
-
103
99
  class PipeTypes:
104
100
  MATERIALIZED = "materialized"
105
101
  ENDPOINT = "endpoint"
@@ -2369,17 +2365,6 @@ def get_project_filenames(folder: str, with_vendor=False) -> List[str]:
2369
2365
  return filenames
2370
2366
 
2371
2367
 
2372
- def get_project_fixtures(folder: str) -> List[str]:
2373
- folders: List[str] = [
2374
- f"{folder}/fixtures/*.ndjson",
2375
- f"{folder}/fixtures/*.csv",
2376
- ]
2377
- filenames: List[str] = []
2378
- for x in folders:
2379
- filenames += glob.glob(x)
2380
- return filenames
2381
-
2382
-
2383
2368
  def has_internal_datafiles(folder: str) -> bool:
2384
2369
  folder = folder or "."
2385
2370
  filenames = get_project_filenames(folder)
@@ -108,10 +108,6 @@ def is_type_date(type_to_check: str) -> bool:
108
108
  return date_type_pattern.match(type_to_check) is not None
109
109
 
110
110
 
111
- def string_test(x: str) -> bool:
112
- return True
113
-
114
-
115
111
  def date_test(x: str) -> bool:
116
112
  return date_pattern.match(x) is not None
117
113
 
@@ -879,6 +879,11 @@ class FeedbackManager:
879
879
  success_delete_rows_datasource = success_message(
880
880
  "** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
881
881
  )
882
+ success_lightweight_delete_rows_datasource = success_message(
883
+ "** Data Source '{datasource}' rows deleted matching condition \"{delete_condition}\""
884
+ "\n Rows affected: {rows_affected}"
885
+ "\n Partitions scanned: {partitions_scanned} (done: {partitions_done}, in progress: {partitions_in_progress})"
886
+ )
882
887
  success_dry_run_delete_rows_datasource = success_message(
883
888
  "** [DRY RUN] Data Source '{datasource}' rows '{rows}' matching condition \"{delete_condition}\" to be deleted"
884
889
  )
@@ -1,5 +1,4 @@
1
1
  from datetime import date
2
- from typing import Optional
3
2
 
4
3
  general_functions = [
5
4
  "BLAKE3",
@@ -400,64 +399,6 @@ Follow the instructions and generate the following response with no additional t
400
399
  """
401
400
 
402
401
 
403
- def create_prompt(existing_resources: str, feedback: str = "", history: str = "") -> str:
404
- feedback_history = ""
405
- if feedback and history:
406
- feedback_history = f"""In case the <feedback> and <history> tags are present and not empty,
407
- it means there was a previous attempt to generate the resources and the user provided feedback and history about previous responses.
408
- Use the following feedback and history to regenerate the response:
409
- Feedback to improve the response:
410
- {feedback}
411
- History of previous results:
412
- {history}"""
413
-
414
- return """
415
- You are a Tinybird expert. You will be given a prompt to generate new or update existing Tinybird resources: datasources and/or pipes.
416
- <existing_resources>{existing_resources}</existing_resources>
417
- {datasource_instructions}
418
- {pipe_instructions}
419
- {sql_instructions}
420
- {datasource_example}
421
- {pipe_example}
422
- {copy_pipe_instructions}
423
- {materialized_pipe_instructions}
424
- {sink_pipe_instructions}
425
- {connection_instructions}
426
- {kafka_connection_example}
427
- {gcs_connection_example}
428
- {gcs_hmac_connection_example}
429
- {s3_connection_example}
430
-
431
- {feedback_history}
432
-
433
- Use the following format to generate the response and do not wrap it in any other text, including the <response> tag.
434
- <response>
435
- <resource>
436
- <type>[datasource or pipe or connection]</type>
437
- <name>[resource name here]</name>
438
- <content>[resource content here]</content>
439
- </resource>
440
- </response>
441
-
442
- """.format(
443
- existing_resources=existing_resources,
444
- datasource_instructions=datasource_instructions,
445
- pipe_instructions=pipe_instructions,
446
- sql_instructions=sql_instructions,
447
- datasource_example=datasource_example,
448
- pipe_example=pipe_example,
449
- copy_pipe_instructions=copy_pipe_instructions,
450
- materialized_pipe_instructions=materialized_pipe_instructions,
451
- sink_pipe_instructions=sink_pipe_instructions,
452
- connection_instructions=connection_instructions,
453
- kafka_connection_example=kafka_connection_example,
454
- gcs_connection_example=gcs_connection_example,
455
- gcs_hmac_connection_example=gcs_hmac_connection_example,
456
- s3_connection_example=s3_connection_example,
457
- feedback_history=feedback_history,
458
- )
459
-
460
-
461
402
  def mock_prompt(rows: int, feedback: str = "") -> str:
462
403
  today = date.today().isoformat()
463
404
 
@@ -824,7 +765,7 @@ Follow these instructions when creating or updating any type of .pipe file:
824
765
  </pipe_file_instructions>
825
766
  """
826
767
 
827
- sql_instructions = """
768
+ sql_instructions = f"""
828
769
  <sql_instructions>
829
770
  - The SQL query must be a valid ClickHouse SQL query that mixes ClickHouse syntax and Tinybird templating syntax (Tornado templating language under the hood).
830
771
  - SQL queries with parameters must start with "%" character and a newline on top of every query to be able to use the parameters. Examples:
@@ -882,186 +823,7 @@ sql_instructions = """
882
823
  - Use the following syntax in the SQL section for the iceberg table function: iceberg('s3://bucket/path/to/table', {{{{tb_secret('aws_access_key_id')}}}}, {{{{tb_secret('aws_secret_access_key')}}}})
883
824
  - Use the following syntax in the SQL section for the postgres table function: postgresql('host:port', 'database', 'table', {{{{tb_secret('db_username')}}}}, {{{{tb_secret('db_password')}}}}), 'schema')
884
825
  </sql_instructions>
885
- """.format(
886
- general_functions=general_functions,
887
- general_functions_insensitive=general_functions_insensitive,
888
- aggregate_functions=aggregate_functions,
889
- )
890
-
891
-
892
- def rules_prompt(source: Optional[str] = None) -> str:
893
- base_command = source or "tb"
894
- return """
895
- You are an expert in SQL and Tinybird. Follow these instructions when working with .datasource and .pipe files:
896
-
897
- <command_calling>
898
- You have commands at your disposal to develop a tinybird project:
899
- - {base_command} build: to build the project locally and check it works.
900
- - {base_command} deployment create --wait --auto: to create a deployment and promote it automatically
901
- - {base_command} test run: to run existing tests
902
- - {base_command} endpoint url <pipe_name>: to get the url of an endpoint, token included.
903
- - {base_command} endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} endpoint data <pipe_name> --param1 value1 --param2 value2
904
- - {base_command} token ls: to list all the tokens
905
- There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
906
- When you need to work with resources or data in cloud, add always the --cloud flag before the command. Example: {base_command} --cloud datasource ls
907
- </command_calling>
908
- <development_instructions>
909
- - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
910
- ├── connections
911
- ├── copies
912
- ├── sinks
913
- ├── datasources
914
- ├── endpoints
915
- ├── fixtures
916
- ├── materializations
917
- ├── pipes
918
- └── tests
919
- - The local development server will be available at http://localhost:7181. Even if some response uses another base url, use always http://localhost:7181.
920
- - After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
921
- - When you need to ingest data locally in a datasource, create a .ndjson file with the same name of the datasource and the data you want and run `{base_command} build` so the data is ingested.
922
- - The format of the generated api endpoint urls is: http://localhost:7181/v0/pipe/<pipe_name>.json?token=<token>
923
- - Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
924
- </development_instructions>
925
- When asking for ingesting data, adding data or appending data do the following depending on the environment you want to work with:
926
- <ingest_data_instructions>
927
- - When building locally, create a .ndjson file with the data you want to ingest and do `{base_command} build` to ingest the data in the build env.
928
- - We call `cloud` the production environment.
929
- - When appending data in cloud, use `{base_command} --cloud datasource append <datasource_name> <file_name>`
930
- - When you have a response that says “there are rows in quarantine”, do `{base_command} [--cloud] datasource data <datasource_name>_quarantine` to understand what is the problem.
931
- </ingest_data_instructions>
932
- <datasource_file_instructions>
933
- Follow these instructions when creating or updating .datasource files:
934
- {datasource_instructions}
935
- </datasource_file_instructions>
936
-
937
- <pipe_file_instructions>
938
- Follow these instructions when creating or updating .pipe files:
939
- {pipe_instructions}
940
- {sql_instructions}
941
- {datasource_example}
942
- {pipe_example}
943
- {copy_pipe_instructions}
944
- {materialized_pipe_instructions}
945
- {sink_pipe_instructions}
946
- {connection_instructions}
947
- {kafka_connection_example}
948
- {gcs_connection_example}
949
- {gcs_hmac_connection_example}
950
- {s3_connection_example}
951
- </pipe_file_instructions>
952
- <test_file_instructions>
953
- Follow these instructions when creating or updating .yaml files for tests:
954
- {test_instructions}
955
- </test_file_instructions>
956
- <deployment_instruction>
957
- Follow these instructions when evolving a datasource schema:
958
- {deployment_instructions}
959
- </deployment_instruction>
960
- """.format(
961
- base_command=base_command,
962
- datasource_instructions=datasource_instructions,
963
- pipe_instructions=pipe_instructions,
964
- sql_instructions=sql_instructions,
965
- datasource_example=datasource_example,
966
- pipe_example=pipe_example,
967
- copy_pipe_instructions=copy_pipe_instructions,
968
- materialized_pipe_instructions=materialized_pipe_instructions,
969
- sink_pipe_instructions=sink_pipe_instructions,
970
- test_instructions=test_instructions,
971
- deployment_instructions=deployment_instructions,
972
- connection_instructions=connection_instructions,
973
- kafka_connection_example=kafka_connection_example,
974
- gcs_connection_example=gcs_connection_example,
975
- gcs_hmac_connection_example=gcs_hmac_connection_example,
976
- s3_connection_example=s3_connection_example,
977
- )
978
-
979
-
980
- def claude_rules_prompt(source: Optional[str] = None) -> str:
981
- base_command = source or "tb"
982
- return """
983
- # Tinybird CLI rules
984
-
985
- ## Commands
986
- You have commands at your disposal to develop a tinybird project:
987
- - {base_command} build: to build the project locally and check it works.
988
- - {base_command} deployment create --wait --auto: to create a deployment and promote it automatically
989
- - {base_command} test run: to run existing tests
990
- - {base_command} endpoint url <pipe_name>: to get the url of an endpoint, token included.
991
- - {base_command} endpoint data <pipe_name>: to get the data of an endpoint. You can pass parameters to the endpoint like this: {base_command} endpoint data <pipe_name> --param1 value1 --param2 value2
992
- - {base_command} token ls: to list all the tokens
993
- There are other commands that you can use, but these are the most common ones. Run `{base_command} -h` to see all the commands if needed.
994
- When you need to work with resources or data in cloud, add always the --cloud flag before the command. Example: {base_command} --cloud datasource ls
995
-
996
- ## Development instructions
997
- - When asking to create a tinybird data project, if the needed folders are not already created, use the following structure:
998
- ├── connections
999
- ├── copies
1000
- ├── sinks
1001
- ├── datasources
1002
- ├── endpoints
1003
- ├── fixtures
1004
- ├── materializations
1005
- ├── pipes
1006
- └── tests
1007
- - The local development server will be available at http://localhost:7181. Even if some response uses another base url, use always http://localhost:7181.
1008
- - After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
1009
- - When you need to ingest data locally in a datasource, create a .ndjson file with the same name of the datasource and the data you want and run `{base_command} build` so the data is ingested.
1010
- - The format of the generated api endpoint urls is: http://localhost:7181/v0/pipe/<pipe_name>.json?token=<token>
1011
- - Before running the tests, remember to have the project built with `{base_command} build` with the latest changes.
1012
- </development_instructions>
1013
- When asking for ingesting data, adding data or appending data do the following depending on the environment you want to work with:
1014
-
1015
- ## Ingestion instructions
1016
- - When building locally, create a .ndjson file with the data you want to ingest and do `{base_command} build` to ingest the data in the build env.
1017
- - We call `cloud` the production environment.
1018
- - When appending data in cloud, use `{base_command} --cloud datasource append <datasource_name> <file_name>`
1019
- - When you have a response that says “there are rows in quarantine”, do `{base_command} [--cloud] datasource data <datasource_name>_quarantine` to understand what is the problem.
1020
-
1021
- ## .datasource file instructions
1022
- Follow these instructions when creating or updating .datasource files:
1023
- {datasource_instructions}
1024
-
1025
- ## .pipe file instructions
1026
- Follow these instructions when creating or updating .pipe files:
1027
- {pipe_instructions}
1028
- {sql_instructions}
1029
- {datasource_example}
1030
- {pipe_example}
1031
- {copy_pipe_instructions}
1032
- {materialized_pipe_instructions}
1033
- {sink_pipe_instructions}
1034
- {connection_instructions}
1035
- {kafka_connection_example}
1036
- {gcs_connection_example}
1037
- {gcs_hmac_connection_example}
1038
- {s3_connection_example}
1039
-
1040
- ## .test file instructions
1041
- Follow these instructions when creating or updating .yaml files for tests:
1042
- {test_instructions}
1043
-
1044
- ## Deployment instructions
1045
- Follow these instructions when evolving a datasource schema:
1046
- {deployment_instructions}
1047
- """.format(
1048
- base_command=base_command,
1049
- datasource_instructions=datasource_instructions,
1050
- pipe_instructions=pipe_instructions,
1051
- sql_instructions=sql_instructions,
1052
- datasource_example=datasource_example,
1053
- pipe_example=pipe_example,
1054
- copy_pipe_instructions=copy_pipe_instructions,
1055
- materialized_pipe_instructions=materialized_pipe_instructions,
1056
- sink_pipe_instructions=sink_pipe_instructions,
1057
- test_instructions=test_instructions,
1058
- deployment_instructions=deployment_instructions,
1059
- connection_instructions=connection_instructions,
1060
- kafka_connection_example=kafka_connection_example,
1061
- gcs_connection_example=gcs_connection_example,
1062
- gcs_hmac_connection_example=gcs_hmac_connection_example,
1063
- s3_connection_example=s3_connection_example,
1064
- )
826
+ """
1065
827
 
1066
828
 
1067
829
  test_instructions = """
@@ -1132,38 +894,6 @@ FORWARD_QUERY >
1132
894
  """
1133
895
 
1134
896
 
1135
- def readme_prompt(readme: str, host: str, token: str, resources_xml: str) -> str:
1136
- return f"""
1137
- You are an expert in SQL and Tinybird. Follow these instructions to generate a new README.md file for a tinybird project:
1138
- Current README.md file:
1139
- <current_resources_xml>
1140
- {resources_xml}
1141
- </current_resources_xml>
1142
- <readme>{readme}</readme>
1143
- <readme_instructions>
1144
- - If it is not present in the current readme, generate a new ## Tinybird section with the following content:
1145
- - ### Overview section:
1146
- - Explaining the purpose of the project.
1147
- - ### Data sources section:
1148
- - Explaining the purpose of each datasource.
1149
- - Add a snippet of how to ingest data into each datasource like the following (where the payload example matches the datasource schema respecting non-nullable types):
1150
- curl -X POST "{host}/v0/events?name=events" \
1151
- -H "Authorization: Bearer {token}" \
1152
- -d '{{"date":"2025-01-31","id":"123","user_id":"abc"}}'
1153
- - ### Endpoints section:
1154
- - Explaining the purpose of each endpoint.
1155
- - Add a snippet of how to use each endpoint like the following:
1156
- curl -X GET "{host}/v0/pipes/pipe_name.json?token={token}"
1157
- - DateTime parameters must be formatted as YYYY-MM-DD HH:MM:SS, or else will fail.
1158
- - Do not include any other extra info related to Tinybird just maintain the existing one in the <readme> tag.
1159
- - Make sure the API host is correct, {host}.
1160
- - It is mandatory to return a <resource> tag with type "readme" in the response.
1161
- - The response must follow the following format:
1162
- <readme>[readme content here]</readme>
1163
- </readme_instructions>
1164
- """
1165
-
1166
-
1167
897
  def quarantine_prompt(datasource_definition: str) -> str:
1168
898
  return f"""
1169
899
  - You are an expert in Tinybird.
@@ -6,7 +6,7 @@ for both Tinybird and Organization scopes.
6
6
  """
7
7
 
8
8
  from functools import lru_cache
9
- from typing import Any, Dict, List, Optional
9
+ from typing import Any, Dict, List
10
10
 
11
11
 
12
12
  def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
@@ -1126,20 +1126,3 @@ def get_service_datasources() -> List[Dict[str, Any]]:
1126
1126
  List[Dict[str, Any]]: A combined list of all service datasource definitions.
1127
1127
  """
1128
1128
  return get_tinybird_service_datasources() + get_organization_service_datasources()
1129
-
1130
-
1131
- def get_service_datasource_by_name(name: str) -> Optional[Dict[str, Any]]:
1132
- """
1133
- Get a specific service datasource by name. Name should include the type (e.g. tinybird.datasources_ops_log)
1134
-
1135
- Args:
1136
- name: The name of the service datasource to retrieve.
1137
-
1138
- Returns:
1139
- Optional[Dict[str, Any]]: The service datasource definition or None if not found.
1140
- """
1141
- service_datasources = get_service_datasources()
1142
- for ds in service_datasources:
1143
- if ds["name"] == name:
1144
- return ds
1145
- return None
@@ -911,30 +911,6 @@ def engine_replicated_to_local(engine: str) -> str:
911
911
  return _RE_REPLICATED_MT.sub(_replace, engine.strip())
912
912
 
913
913
 
914
- def engine_patch_replicated_engine(engine: str, engine_full: Optional[str], new_table_name: str) -> Optional[str]:
915
- """
916
- >>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192", 'table_name_staging')
917
- "ReplicatedMergeTree('/clickhouse/tables/1-1/table_name_staging', 'replica') PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192"
918
- >>> engine_patch_replicated_engine("ReplicatedMergeTree", "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)", 'sales_product_rank_rt_replicated_2_staging')
919
- "ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/sales_product_rank_rt_replicated_2_staging', '{replica}') PARTITION BY toYYYYMM(date) ORDER BY (purchase_location, sku_rank_lc, date)"
920
- >>> engine_patch_replicated_engine("ReplicatedMergeTree", None, 't_000') is None
921
- True
922
- >>> engine_patch_replicated_engine("Log", "Log()", 't_000')
923
- 'Log()'
924
- >>> engine_patch_replicated_engine("MergeTree", "MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024", 't_000')
925
- 'MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024'
926
- """
927
- if not engine_full:
928
- return None
929
- if engine.lower().startswith("Replicated".lower()):
930
- parts = _RE_REPLICATED_ENGINE_SPLIT.split(engine_full)
931
- paths = parts[2].split("/")
932
- paths[-1] = new_table_name
933
- zoo_path = "/".join(paths)
934
- return "".join([*parts[:2], zoo_path, *parts[3:]])
935
- return engine_full
936
-
937
-
938
914
  if __name__ == "__main__":
939
915
  print( # noqa: T201
940
916
  _parse_table_structure(
@@ -1542,35 +1542,6 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
1542
1542
  raise e
1543
1543
 
1544
1544
 
1545
- class CodeWriter:
1546
- def __init__(self, file, template):
1547
- self.file = file
1548
- self.current_template = template
1549
- self.apply_counter = 0
1550
- self._indent = 0
1551
-
1552
- def indent_size(self):
1553
- return self._indent
1554
-
1555
- def indent(self):
1556
- class Indenter:
1557
- def __enter__(_):
1558
- self._indent += 1
1559
- return self
1560
-
1561
- def __exit__(_, *args):
1562
- assert self._indent > 0
1563
- self._indent -= 1
1564
-
1565
- return Indenter()
1566
-
1567
- def write_line(self, line, line_number, indent=None):
1568
- if indent is None:
1569
- indent = self._indent
1570
- line_comment = " # %s:%d" % ("<generated>", line_number)
1571
- print(" " * indent + line + line_comment, file=self.file)
1572
-
1573
-
1574
1545
  def get_var_names(t: Template):
1575
1546
  """
1576
1547
  Extract variable names from a template.
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/forward/commands'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '4.6.4'
8
- __revision__ = 'cad29af'
7
+ __version__ = '4.6.5'
8
+ __revision__ = '939e465'
@@ -3,6 +3,7 @@ import logging
3
3
  import os
4
4
  import ssl
5
5
  import time
6
+ from dataclasses import dataclass
6
7
  from pathlib import Path
7
8
  from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Tuple, Union
8
9
  from urllib.parse import parse_qsl, quote, urlencode, urlsplit
@@ -20,6 +21,19 @@ LAST_PARTITION = "last_partition"
20
21
  ALL_PARTITIONS = "all_partitions"
21
22
 
22
23
 
24
+ @dataclass(frozen=True)
25
+ class CLIWarning:
26
+ message: str
27
+ code: Optional[str] = None
28
+ docs_url: Optional[str] = None
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class CLIResourceFile:
33
+ content: str
34
+ warnings: list[CLIWarning]
35
+
36
+
23
37
  class AuthException(Exception):
24
38
  pass
25
39
 
@@ -376,8 +390,22 @@ class TinyB:
376
390
  def pipe_file(self, pipe: str):
377
391
  return self._req(f"/v1/pipes/{pipe}.pipe")
378
392
 
379
- def connection_file(self, connection: str):
380
- return self._req(f"/v0/connectors/{connection}.connection")
393
+ def _parse_cli_warnings(self, warnings: list[dict[str, Any]]) -> list[CLIWarning]:
394
+ return [
395
+ CLIWarning(
396
+ message=warning["message"],
397
+ code=warning.get("code"),
398
+ docs_url=warning.get("docs_url"),
399
+ )
400
+ for warning in warnings
401
+ ]
402
+
403
+ def connection_file(self, connection: str) -> CLIResourceFile:
404
+ response = self._req(f"/v0/connectors/{connection}.connection?include_metadata=true")
405
+ return CLIResourceFile(
406
+ content=response["content"],
407
+ warnings=self._parse_cli_warnings(response.get("warnings", [])),
408
+ )
381
409
 
382
410
  def datasource_file(self, datasource: str):
383
411
  try:
@@ -91,12 +91,6 @@ def get_config(
91
91
  return config
92
92
 
93
93
 
94
- def write_config(config: Dict[str, Any], dest_file: str = ".tinyb") -> None:
95
- config_file = Path(getcwd()) / dest_file
96
- with open(config_file, "w") as file:
97
- file.write(json.dumps(config, indent=4, sort_keys=True))
98
-
99
-
100
94
  def get_display_cloud_host(api_host: str) -> str:
101
95
  is_local = "localhost" in api_host
102
96
  if is_local: