tinybird 0.0.1.dev149__tar.gz → 0.0.1.dev152__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (109) hide show
  1. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/PKG-INFO +1 -1
  2. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/__cli__.py +2 -2
  3. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/cli.py +10 -3
  4. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/create.py +54 -50
  5. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/deprecations.py +7 -0
  6. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/local_common.py +19 -5
  7. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/mock.py +58 -30
  8. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/project.py +9 -0
  9. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/test.py +33 -5
  10. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird.egg-info/PKG-INFO +1 -1
  11. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/setup.cfg +0 -0
  12. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/__cli__.py +0 -0
  13. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/ch_utils/constants.py +0 -0
  14. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/ch_utils/engine.py +0 -0
  15. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/check_pypi.py +0 -0
  16. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/client.py +0 -0
  17. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/config.py +0 -0
  18. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/connectors.py +0 -0
  19. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/context.py +0 -0
  20. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/datafile.py +0 -0
  21. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/datatypes.py +0 -0
  22. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/feedback_manager.py +0 -0
  23. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/git_settings.py +0 -0
  24. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/prompts.py +0 -0
  25. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/sql.py +0 -0
  26. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/sql_template.py +0 -0
  27. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/sql_template_fmt.py +0 -0
  28. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/sql_toolset.py +0 -0
  29. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/syncasync.py +0 -0
  30. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/check_pypi.py +0 -0
  31. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/cli.py +0 -0
  32. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/client.py +0 -0
  33. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/config.py +0 -0
  34. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/auth.py +0 -0
  35. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/build.py +0 -0
  36. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/cicd.py +0 -0
  37. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/common.py +0 -0
  38. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/config.py +0 -0
  39. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/connection.py +0 -0
  40. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/copy.py +0 -0
  41. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/build.py +0 -0
  42. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/build_common.py +0 -0
  43. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  44. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  45. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/common.py +0 -0
  46. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/diff.py +0 -0
  47. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  48. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/fixture.py +0 -0
  49. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/format_common.py +0 -0
  50. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  51. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  52. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
  53. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
  54. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  55. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/playground.py +0 -0
  56. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datafile/pull.py +0 -0
  57. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/datasource.py +0 -0
  58. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/deployment.py +0 -0
  59. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/endpoint.py +0 -0
  60. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/exceptions.py +0 -0
  61. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/feedback_manager.py +0 -0
  62. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/infra.py +0 -0
  63. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/job.py +0 -0
  64. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/llm.py +0 -0
  65. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/llm_utils.py +0 -0
  66. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/local.py +0 -0
  67. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/login.py +0 -0
  68. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/logout.py +0 -0
  69. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/materialization.py +0 -0
  70. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/open.py +0 -0
  71. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/pipe.py +0 -0
  72. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/regions.py +0 -0
  73. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/secret.py +0 -0
  74. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/shell.py +0 -0
  75. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/table.py +0 -0
  76. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/telemetry.py +0 -0
  77. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  78. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  79. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/token.py +0 -0
  80. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/watch.py +0 -0
  81. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/workspace.py +0 -0
  82. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb/modules/workspace_members.py +0 -0
  83. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli.py +0 -0
  84. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/auth.py +0 -0
  85. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/branch.py +0 -0
  86. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/cicd.py +0 -0
  87. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/cli.py +0 -0
  88. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/common.py +0 -0
  89. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/config.py +0 -0
  90. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/connection.py +0 -0
  91. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/datasource.py +0 -0
  92. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/exceptions.py +0 -0
  93. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/fmt.py +0 -0
  94. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/job.py +0 -0
  95. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/pipe.py +0 -0
  96. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/regions.py +0 -0
  97. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/tag.py +0 -0
  98. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/telemetry.py +0 -0
  99. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/test.py +0 -0
  100. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  101. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  102. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/workspace.py +0 -0
  103. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  104. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird/tornado_template.py +0 -0
  105. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird.egg-info/SOURCES.txt +0 -0
  106. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird.egg-info/dependency_links.txt +0 -0
  107. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird.egg-info/entry_points.txt +0 -0
  108. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird.egg-info/requires.txt +0 -0
  109. {tinybird-0.0.1.dev149 → tinybird-0.0.1.dev152}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev149
3
+ Version: 0.0.1.dev152
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev149'
8
- __revision__ = '2838b1f'
7
+ __version__ = '0.0.1.dev152'
8
+ __revision__ = '100f32b'
@@ -323,16 +323,23 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, s
323
323
  return None
324
324
 
325
325
  commands_always_cloud = ["pull", "infra"]
326
- commands_always_local = ["build", "test", "dev"]
327
- if (cloud or command in commands_always_cloud) and command not in commands_always_local:
326
+ commands_always_local = ["build", "dev"]
327
+ command_always_test = ["test"]
328
+
329
+ if (
330
+ (cloud or command in commands_always_cloud)
331
+ and command not in commands_always_local
332
+ and command not in command_always_test
333
+ ):
328
334
  click.echo(
329
335
  FeedbackManager.gray(message=f"Running against Tinybird Cloud: Workspace {config.get('name', 'default')}")
330
336
  )
331
337
  return _get_tb_client(config.get("token", None), config["host"], staging=staging)
332
338
  local = command in commands_always_local
339
+ test = command in command_always_test
333
340
  if not local and command not in commands_always_local and command:
334
341
  click.echo(FeedbackManager.gray(message="Running against Tinybird Local"))
335
- return await get_tinybird_local_client(config, staging=staging)
342
+ return await get_tinybird_local_client(config, test=test, staging=staging)
336
343
 
337
344
 
338
345
  def get_target_env(cloud: bool) -> str:
@@ -1,3 +1,4 @@
1
+ import glob
1
2
  import os
2
3
  import re
3
4
  from pathlib import Path
@@ -5,7 +6,7 @@ from typing import Any, Dict, Optional, Tuple
5
6
 
6
7
  import click
7
8
 
8
- from tinybird.prompts import create_prompt, mock_prompt, readme_prompt, rules_prompt
9
+ from tinybird.prompts import create_prompt, readme_prompt, rules_prompt
9
10
  from tinybird.tb.client import TinyB
10
11
  from tinybird.tb.modules.cicd import init_cicd
11
12
  from tinybird.tb.modules.cli import cli
@@ -17,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
17
18
  from tinybird.tb.modules.llm import LLM
18
19
  from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
19
20
  from tinybird.tb.modules.local_common import get_tinybird_local_client
21
+ from tinybird.tb.modules.mock import create_mock_data
20
22
  from tinybird.tb.modules.project import Project
21
23
 
22
24
 
@@ -83,7 +85,7 @@ async def create(
83
85
 
84
86
  if data or (prompt and user_token):
85
87
  click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
86
- result, created_something = await create_resources(tb_client, user_token, data, prompt, folder, ctx_config)
88
+ result, created_something = await create_resources(tb_client, user_token, data, prompt, project, ctx_config)
87
89
  click.echo(FeedbackManager.success(message="✓ Done!\n"))
88
90
  if prompt:
89
91
  readme_path = Path(root_folder) / "README.md"
@@ -132,22 +134,26 @@ async def create(
132
134
  persist_fixture(ds_name, data_content, folder, format=data_format)
133
135
  created_something = True
134
136
  elif prompt and user_token:
135
- datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
136
- for datasource_file in datasource_files:
137
- datasource_path = Path(folder) / "datasources" / datasource_file
138
- llm = LLM(user_token=user_token, host=tb_client.host)
137
+ for datasource_file in project.get_datasource_files():
138
+ datasource_path = Path(datasource_file)
139
139
  datasource_name = datasource_path.stem
140
140
  datasource_content = datasource_path.read_text()
141
141
  has_json_path = "`json:" in datasource_content
142
142
  if has_json_path:
143
- prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
144
- response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
145
- sql = extract_xml(response, "sql")
146
- sql = sql.split("FORMAT")[0]
147
- query_result = await tb_client.query(f"{sql} FORMAT JSON")
148
- data = query_result.get("data", [])
149
- if data:
150
- persist_fixture(datasource_name, data, folder, format="ndjson")
143
+ mock_data = await create_mock_data(
144
+ datasource_name,
145
+ datasource_content,
146
+ rows,
147
+ prompt,
148
+ config,
149
+ ctx_config,
150
+ user_token,
151
+ tb_client,
152
+ format_="ndjson",
153
+ folder=project.folder,
154
+ )
155
+ if mock_data:
156
+ persist_fixture(datasource_name, mock_data, folder, format="ndjson")
151
157
  click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
152
158
  created_something = True
153
159
 
@@ -204,12 +210,12 @@ async def create_resources(
204
210
  user_token: Optional[str],
205
211
  data: Optional[str],
206
212
  prompt: Optional[str],
207
- folder: str,
213
+ project: Project,
208
214
  config: Dict[str, Any],
209
215
  ) -> Tuple[str, bool]:
210
216
  result = ""
211
217
  created_any_resource = False
212
- folder_path = Path(folder)
218
+ folder_path = project.path
213
219
  if data:
214
220
  local_client = await get_tinybird_local_client(config)
215
221
  path = folder_path / data
@@ -224,7 +230,7 @@ SQL >
224
230
  SELECT * from {name}
225
231
  TYPE ENDPOINT
226
232
  """,
227
- folder,
233
+ project.folder,
228
234
  )
229
235
  result = (
230
236
  f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
@@ -232,19 +238,9 @@ TYPE ENDPOINT
232
238
  created_any_resource = True
233
239
 
234
240
  elif prompt and user_token:
235
- datasource_paths = [
236
- Path(folder) / "datasources" / f
237
- for f in os.listdir(Path(folder) / "datasources")
238
- if f.endswith(".datasource")
239
- ]
240
- pipes_paths = [
241
- Path(folder) / "endpoints" / f for f in os.listdir(Path(folder) / "endpoints") if f.endswith(".pipe")
242
- ]
243
- connections_paths = [
244
- Path(folder) / "connections" / f
245
- for f in os.listdir(Path(folder) / "connections")
246
- if f.endswith(".connection")
247
- ]
241
+ datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
242
+ pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
243
+ connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
248
244
  resources_xml = "\n".join(
249
245
  [
250
246
  f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
@@ -300,19 +296,19 @@ TYPE ENDPOINT
300
296
  data=None,
301
297
  _format="ndjson",
302
298
  force=True,
303
- folder=folder,
299
+ folder=project.folder,
304
300
  )
305
301
  created_any_resource = True
306
302
 
307
303
  for pipe in pipes:
308
304
  content = pipe["content"].replace("```", "")
309
- generate_pipe_file(pipe["name"], content, folder)
305
+ generate_pipe_file(pipe["name"], content, project.folder)
310
306
  created_any_resource = True
311
307
 
312
308
  for conn in connections:
313
309
  content = conn["content"].replace("```", "")
314
310
  filename = f"{conn['name']}.connection"
315
- generate_connection_file(conn["name"], content, folder)
311
+ generate_connection_file(conn["name"], content, project.folder)
316
312
  created_any_resource = True
317
313
 
318
314
  return result, created_any_resource
@@ -348,21 +344,25 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
348
344
  def is_endpoint(content: str) -> bool:
349
345
  return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
350
346
 
351
- if is_copy(content):
352
- pathname = "copies"
353
- elif is_materialization(content):
354
- pathname = "materializations"
355
- elif is_sink(content):
356
- pathname = "sinks"
357
- elif is_endpoint(content):
358
- pathname = "endpoints"
347
+ already_exists = glob.glob(f"{folder}/**/{name}.pipe")
348
+ if already_exists:
349
+ f = Path(already_exists[0])
359
350
  else:
360
- pathname = "pipes"
351
+ if is_copy(content):
352
+ pathname = "copies"
353
+ elif is_materialization(content):
354
+ pathname = "materializations"
355
+ elif is_sink(content):
356
+ pathname = "sinks"
357
+ elif is_endpoint(content):
358
+ pathname = "endpoints"
359
+ else:
360
+ pathname = "pipes"
361
361
 
362
- base = Path(folder) / pathname
363
- if not base.exists():
364
- base.mkdir()
365
- f = base / (f"{name}.pipe")
362
+ base = Path(folder) / pathname
363
+ if not base.exists():
364
+ base.mkdir()
365
+ f = base / (f"{name}.pipe")
366
366
  with open(f"{f}", "w") as file:
367
367
  file.write(content)
368
368
  click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
@@ -370,10 +370,14 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
370
370
 
371
371
 
372
372
  def generate_connection_file(name: str, content: str, folder: str, skip_feedback: bool = False) -> Path:
373
- base = Path(folder) / "connections"
374
- if not base.exists():
375
- base.mkdir()
376
- f = base / (f"{name}.connection")
373
+ already_exists = glob.glob(f"{folder}/**/{name}.connection")
374
+ if already_exists:
375
+ f = Path(already_exists[0])
376
+ else:
377
+ base = Path(folder) / "connections"
378
+ if not base.exists():
379
+ base.mkdir()
380
+ f = base / (f"{name}.connection")
377
381
  with open(f"{f}", "w") as file:
378
382
  file.write(content)
379
383
  if not skip_feedback:
@@ -9,6 +9,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
9
9
  context_settings=dict(
10
10
  ignore_unknown_options=True,
11
11
  ),
12
+ hidden=True,
12
13
  )
13
14
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
14
15
  def branch(args) -> None:
@@ -30,6 +31,7 @@ def branch(args) -> None:
30
31
  context_settings=dict(
31
32
  ignore_unknown_options=True,
32
33
  ),
34
+ hidden=True,
33
35
  )
34
36
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
35
37
  def check(args) -> None:
@@ -47,6 +49,7 @@ def check(args) -> None:
47
49
  context_settings=dict(
48
50
  ignore_unknown_options=True,
49
51
  ),
52
+ hidden=True,
50
53
  )
51
54
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
52
55
  def diff(args) -> None:
@@ -64,6 +67,7 @@ def diff(args) -> None:
64
67
  context_settings=dict(
65
68
  ignore_unknown_options=True,
66
69
  ),
70
+ hidden=True,
67
71
  )
68
72
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
69
73
  def fmt(args) -> None:
@@ -81,6 +85,7 @@ def fmt(args) -> None:
81
85
  context_settings=dict(
82
86
  ignore_unknown_options=True,
83
87
  ),
88
+ hidden=True,
84
89
  )
85
90
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
86
91
  def init(args) -> None:
@@ -98,6 +103,7 @@ def init(args) -> None:
98
103
  context_settings=dict(
99
104
  ignore_unknown_options=True,
100
105
  ),
106
+ hidden=True,
101
107
  )
102
108
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
103
109
  def push(args) -> None:
@@ -115,6 +121,7 @@ def push(args) -> None:
115
121
  context_settings=dict(
116
122
  ignore_unknown_options=True,
117
123
  ),
124
+ hidden=True,
118
125
  )
119
126
  @click.argument("args", nargs=-1, type=click.UNPROCESSED)
120
127
  def tag(args) -> None:
@@ -18,14 +18,14 @@ TB_LOCAL_HOST = re.sub(r"^https?://", "", os.getenv("TB_LOCAL_HOST", "localhost"
18
18
  TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
19
19
 
20
20
 
21
- async def get_tinybird_local_client(config_obj: Dict[str, Any], staging: bool = False) -> TinyB:
21
+ async def get_tinybird_local_client(config_obj: Dict[str, Any], test: bool = False, staging: bool = False) -> TinyB:
22
22
  """Get a Tinybird client connected to the local environment."""
23
23
 
24
- config = await get_tinybird_local_config(config_obj)
24
+ config = await get_tinybird_local_config(config_obj, test=test)
25
25
  return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
26
26
 
27
27
 
28
- async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
28
+ async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False) -> CLIConfig:
29
29
  """Craft a client config with a workspace name based on the path of the project files
30
30
 
31
31
  It uses the tokens from tinybird local
@@ -47,7 +47,10 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
47
47
  # Create a new workspace if path is provided. This is used to isolate the build in a different workspace.
48
48
  if path:
49
49
  user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
50
- ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
50
+ if test:
51
+ ws_name = get_test_workspace_name(path)
52
+ else:
53
+ ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
51
54
  if not ws_name:
52
55
  raise AuthNoTokenException()
53
56
 
@@ -61,6 +64,13 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
61
64
 
62
65
  ws = next((ws for ws in local_workspaces if ws["name"] == ws_name), None)
63
66
 
67
+ # If we are running a test, we need to delete the workspace if it already exists
68
+ if test and ws:
69
+ requests.delete(
70
+ f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
71
+ )
72
+ ws = None
73
+
64
74
  if not ws:
65
75
  await user_client.create_workspace(
66
76
  ws_name, template=None, assign_to_organization_id=user_org_id, version="v1"
@@ -71,7 +81,6 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
71
81
  raise AuthNoTokenException()
72
82
 
73
83
  ws_token = ws["token"]
74
-
75
84
  config.set_token(ws_token)
76
85
  config.set_token_for_host(TB_LOCAL_ADDRESS, ws_token)
77
86
  config.set_host(TB_LOCAL_ADDRESS)
@@ -86,3 +95,8 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any]) -> CLIConfig:
86
95
  def get_build_workspace_name(path: str) -> str:
87
96
  folder_hash = hashlib.sha256(path.encode()).hexdigest()
88
97
  return f"Tinybird_Local_Build_{folder_hash}"
98
+
99
+
100
+ def get_test_workspace_name(path: str) -> str:
101
+ folder_hash = hashlib.sha256(path.encode()).hexdigest()
102
+ return f"Tinybird_Local_Test_{folder_hash}"
@@ -1,5 +1,6 @@
1
1
  import glob
2
2
  from pathlib import Path
3
+ from typing import Any, Dict, List
3
4
 
4
5
  import click
5
6
 
@@ -65,41 +66,23 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
65
66
 
66
67
  datasource_content = datasource_path.read_text()
67
68
  config = CLIConfig.get_project_config()
68
- user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
69
69
  user_token = ctx_config.get("user_token")
70
70
 
71
71
  if not user_token:
72
72
  raise Exception("This action requires authentication. Run 'tb login' first.")
73
73
 
74
- llm = LLM(user_token=user_token, host=user_client.host)
75
- prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
76
- sql = ""
77
- attempts = 0
78
- data = []
79
- error = ""
80
- sql_path = None
81
- while True:
82
- try:
83
- response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
84
- sql = extract_xml(response, "sql")
85
- sql_path = persist_fixture_sql(datasource_name, sql, folder)
86
- sql_format = "JSON" if format_ == "ndjson" else "CSV"
87
- result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
88
- if sql_format == "JSON":
89
- data = result.get("data", [])[:rows]
90
- error_response = result.get("error", None)
91
- if error_response:
92
- raise Exception(error_response)
93
- else:
94
- data = result
95
- break
96
- except Exception as e:
97
- error = str(e)
98
- attempts += 1
99
- if attempts > 5:
100
- raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
101
- else:
102
- continue
74
+ data = await create_mock_data(
75
+ datasource_name,
76
+ datasource_content,
77
+ rows,
78
+ prompt,
79
+ config,
80
+ ctx_config,
81
+ user_token,
82
+ tb_client,
83
+ format_,
84
+ folder,
85
+ )
103
86
 
104
87
  fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
105
88
  click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
@@ -125,3 +108,48 @@ async def append_fixture(
125
108
  concurrency=1,
126
109
  silent=True,
127
110
  )
111
+
112
+
113
+ async def create_mock_data(
114
+ datasource_name: str,
115
+ datasource_content: str,
116
+ rows: int,
117
+ prompt: str,
118
+ config: CLIConfig,
119
+ ctx_config: Dict[str, Any],
120
+ user_token: str,
121
+ tb_client: TinyB,
122
+ format_: str,
123
+ folder: str,
124
+ ) -> List[Dict[str, Any]]:
125
+ user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
126
+ llm = LLM(user_token=user_token, host=user_client.host)
127
+ prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
128
+ sql = ""
129
+ attempts = 0
130
+ data = []
131
+ error = ""
132
+ sql_path = None
133
+ while True:
134
+ try:
135
+ response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt)
136
+ sql = extract_xml(response, "sql")
137
+ sql_path = persist_fixture_sql(datasource_name, sql, folder)
138
+ sql_format = "JSON" if format_ == "ndjson" else "CSV"
139
+ result = await tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
140
+ if sql_format == "JSON":
141
+ data = result.get("data", [])[:rows]
142
+ error_response = result.get("error", None)
143
+ if error_response:
144
+ raise Exception(error_response)
145
+ else:
146
+ data = result
147
+ break
148
+ except Exception as e:
149
+ error = str(e)
150
+ attempts += 1
151
+ if attempts > 5:
152
+ raise Exception(f"Failed to generate a valid solution. Check {str(sql_path)} and try again.")
153
+ else:
154
+ continue
155
+ return data
@@ -55,6 +55,15 @@ class Project:
55
55
  def connections(self) -> List[str]:
56
56
  return sorted([Path(f).stem for f in glob.glob(f"{self.path}/**/*.connection", recursive=False)])
57
57
 
58
+ def get_datasource_files(self) -> List[str]:
59
+ return glob.glob(f"{self.path}/**/*.datasource", recursive=False)
60
+
61
+ def get_pipe_files(self) -> List[str]:
62
+ return glob.glob(f"{self.path}/**/*.pipe", recursive=False)
63
+
64
+ def get_connection_files(self) -> List[str]:
65
+ return glob.glob(f"{self.path}/**/*.connection", recursive=False)
66
+
58
67
  def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
59
68
  try:
60
69
  return parse_pipe(filename).datafile
@@ -8,22 +8,25 @@ import difflib
8
8
  import glob
9
9
  import sys
10
10
  import urllib.parse
11
+ from copy import deepcopy
11
12
  from pathlib import Path
12
13
  from typing import Any, Dict, List, Optional, Tuple
13
14
 
14
15
  import click
16
+ import requests
15
17
  import yaml
16
18
  from requests import Response
17
19
 
18
20
  from tinybird.prompts import test_create_prompt
19
21
  from tinybird.tb.client import TinyB
20
- from tinybird.tb.modules.build import process
22
+ from tinybird.tb.modules.build import process as build_project
21
23
  from tinybird.tb.modules.cli import cli
22
24
  from tinybird.tb.modules.config import CLIConfig
23
- from tinybird.tb.modules.exceptions import CLITestException
25
+ from tinybird.tb.modules.exceptions import CLILocalException, CLITestException
24
26
  from tinybird.tb.modules.feedback_manager import FeedbackManager
25
27
  from tinybird.tb.modules.llm import LLM
26
28
  from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
29
+ from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_test_workspace_name
27
30
  from tinybird.tb.modules.project import Project
28
31
 
29
32
  yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
@@ -77,7 +80,7 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
77
80
  project: Project = ctx.ensure_object(dict)["project"]
78
81
  client: TinyB = ctx.ensure_object(dict)["client"]
79
82
  click.echo(FeedbackManager.highlight(message="\n» Building project"))
80
- process(project=project, tb_client=client, watch=False, silent=True)
83
+ build_project(project=project, tb_client=client, watch=False, silent=True)
81
84
  click.echo(FeedbackManager.info(message="✓ Done!\n"))
82
85
  config = CLIConfig.get_project_config()
83
86
  folder = project.folder
@@ -140,6 +143,8 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
140
143
  click.echo(FeedbackManager.success(message="✓ Done!\n"))
141
144
  except Exception as e:
142
145
  raise CLITestException(FeedbackManager.error(message=str(e)))
146
+ finally:
147
+ cleanup_test_workspace(client, project.folder)
143
148
 
144
149
 
145
150
  @test.command(
@@ -154,7 +159,7 @@ def test_update(ctx: click.Context, pipe: str) -> None:
154
159
  project: Project = ctx.ensure_object(dict)["project"]
155
160
  folder = project.folder
156
161
  click.echo(FeedbackManager.highlight(message="\n» Building project"))
157
- process(project=project, tb_client=client, watch=False, silent=True)
162
+ build_project(project=project, tb_client=client, watch=False, silent=True)
158
163
  click.echo(FeedbackManager.info(message="✓ Done!"))
159
164
  pipe_tests_path = get_pipe_path(pipe, folder)
160
165
  pipe_name = pipe_tests_path.stem
@@ -191,6 +196,8 @@ def test_update(ctx: click.Context, pipe: str) -> None:
191
196
  click.echo(FeedbackManager.success(message="✓ Done!\n"))
192
197
  except Exception as e:
193
198
  raise CLITestException(FeedbackManager.error(message=str(e)))
199
+ finally:
200
+ cleanup_test_workspace(client, project.folder)
194
201
 
195
202
 
196
203
  @test.command(
@@ -204,7 +211,7 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
204
211
  client: TinyB = ctx.ensure_object(dict)["client"]
205
212
  project: Project = ctx.ensure_object(dict)["project"]
206
213
  click.echo(FeedbackManager.highlight(message="\n» Building project"))
207
- process(project=project, tb_client=client, watch=False, silent=True)
214
+ build_project(project=project, tb_client=client, watch=False, silent=True)
208
215
  click.echo(FeedbackManager.info(message="✓ Done!"))
209
216
 
210
217
  click.echo(FeedbackManager.highlight(message="\n» Running tests"))
@@ -268,6 +275,8 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
268
275
  click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
269
276
  except Exception as e:
270
277
  raise CLITestException(FeedbackManager.error(message=str(e)))
278
+ finally:
279
+ cleanup_test_workspace(client, project.folder)
271
280
 
272
281
 
273
282
  async def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
@@ -301,3 +310,22 @@ def get_pipe_path(name_or_filename: str, folder: str) -> Path:
301
310
  raise Exception(f"Pipe {name_or_filename} not found")
302
311
 
303
312
  return pipe_path
313
+
314
+
315
+ def cleanup_test_workspace(client: TinyB, path: str) -> None:
316
+ user_client = deepcopy(client)
317
+ try:
318
+ # ruff: noqa: ASYNC210
319
+ tokens = requests.get(f"{TB_LOCAL_ADDRESS}/tokens").json()
320
+ except Exception:
321
+ raise CLILocalException(
322
+ FeedbackManager.error(message="Tinybird local is not running. Please run `tb local start` first.")
323
+ )
324
+ try:
325
+ user_token = tokens["user_token"]
326
+ user_client.token = user_token
327
+ asyncio.run(
328
+ user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
329
+ )
330
+ except Exception:
331
+ pass
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev149
3
+ Version: 0.0.1.dev152
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/cli/introduction.html
6
6
  Author: Tinybird