tinybird 0.0.1.dev43__py3-none-any.whl → 0.0.1.dev46__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. tinybird/client.py +17 -1
  2. tinybird/prompts.py +135 -15
  3. tinybird/tb/__cli__.py +2 -2
  4. tinybird/tb/cli.py +1 -1
  5. tinybird/tb/modules/build.py +28 -20
  6. tinybird/tb/modules/cli.py +18 -62
  7. tinybird/tb/modules/common.py +3 -2
  8. tinybird/tb/modules/copy.py +1 -1
  9. tinybird/tb/modules/create.py +134 -59
  10. tinybird/tb/modules/datafile/build.py +12 -221
  11. tinybird/tb/modules/datafile/common.py +1 -1
  12. tinybird/tb/modules/datafile/format_datasource.py +1 -1
  13. tinybird/tb/modules/datafile/format_pipe.py +4 -4
  14. tinybird/tb/modules/datafile/pipe_checker.py +3 -3
  15. tinybird/tb/modules/datasource.py +1 -1
  16. tinybird/tb/modules/deployment.py +1 -1
  17. tinybird/tb/modules/endpoint.py +89 -2
  18. tinybird/tb/modules/feedback_manager.py +5 -1
  19. tinybird/tb/modules/local_common.py +10 -7
  20. tinybird/tb/modules/materialization.py +146 -0
  21. tinybird/tb/modules/mock.py +56 -16
  22. tinybird/tb/modules/pipe.py +8 -326
  23. tinybird/tb/modules/project.py +10 -4
  24. tinybird/tb/modules/shell.py +3 -3
  25. tinybird/tb/modules/test.py +73 -38
  26. tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
  27. tinybird/tb/modules/update.py +1 -1
  28. tinybird/tb/modules/workspace.py +2 -1
  29. {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/METADATA +1 -1
  30. {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/RECORD +33 -33
  31. tinybird/tb/modules/build_client.py +0 -199
  32. {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/WHEEL +0 -0
  33. {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/entry_points.txt +0 -0
  34. {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/top_level.txt +0 -0
@@ -7,7 +7,7 @@ from typing import Optional
7
7
  import click
8
8
 
9
9
  from tinybird.client import TinyB
10
- from tinybird.prompts import create_prompt, cursorrules_prompt, mock_prompt
10
+ from tinybird.prompts import create_prompt, mock_prompt, rules_prompt
11
11
  from tinybird.tb.modules.cicd import init_cicd
12
12
  from tinybird.tb.modules.cli import cli
13
13
  from tinybird.tb.modules.common import _generate_datafile, check_user_token_with_client, coro, generate_datafile
@@ -18,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
18
18
  from tinybird.tb.modules.llm import LLM
19
19
  from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
20
20
  from tinybird.tb.modules.local_common import get_tinybird_local_client
21
+ from tinybird.tb.modules.project import Project
21
22
 
22
23
 
23
24
  @cli.command()
@@ -35,28 +36,33 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
35
36
  )
36
37
  @click.option(
37
38
  "--folder",
38
- default=".",
39
+ default="",
39
40
  type=click.Path(exists=False, file_okay=False),
40
41
  help="Folder where datafiles will be placed",
41
42
  )
42
43
  @click.option("--rows", type=int, default=10, help="Number of events to send")
43
44
  @click.option("--cursor", default=False, is_flag=True, help="Create .cursorrules file with Tinybird rules")
45
+ @click.option("--source", type=str, default="tb", help="Source of the command")
46
+ @click.pass_context
44
47
  @coro
45
48
  async def create(
49
+ ctx: click.Context,
46
50
  data: Optional[str],
47
51
  prompt: Optional[str],
48
52
  folder: Optional[str],
49
53
  rows: int,
50
54
  cursor: bool,
55
+ source: str,
51
56
  ) -> None:
52
57
  """Initialize a new project."""
58
+ project: Project = ctx.ensure_object(dict)["project"]
53
59
  folder = folder or getcwd()
54
60
  folder_path = Path(folder)
55
61
  if not folder_path.exists():
56
62
  folder_path.mkdir()
57
63
 
58
64
  try:
59
- config = CLIConfig.get_project_config(folder)
65
+ config = CLIConfig.get_project_config(str(project.path))
60
66
  tb_client = config.get_client()
61
67
  user_token: Optional[str] = None
62
68
  if prompt:
@@ -79,9 +85,11 @@ async def create(
79
85
  create_project_structure(folder)
80
86
  click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
81
87
 
82
- click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
83
- datasources_created = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
84
- click.echo(FeedbackManager.success(message=" Done!\n"))
88
+ result = ""
89
+ if data or prompt:
90
+ click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
91
+ result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
92
+ click.echo(FeedbackManager.success(message="✓ Done!\n"))
85
93
 
86
94
  if not already_has_cicd(folder):
87
95
  click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
@@ -89,7 +97,7 @@ async def create(
89
97
  await init_cicd(data_project_dir=os.path.relpath(folder))
90
98
  click.echo(FeedbackManager.success(message="✓ Done!\n"))
91
99
 
92
- if validate_fixtures(folder) and datasources_created:
100
+ if should_generate_fixtures(result):
93
101
  click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
94
102
 
95
103
  if data:
@@ -110,12 +118,12 @@ async def create(
110
118
  datasource_content = datasource_path.read_text()
111
119
  has_json_path = "`json:" in datasource_content
112
120
  if has_json_path:
113
- prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
121
+ prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
114
122
  response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
115
123
  sql = extract_xml(response, "sql")
116
124
  sql = sql.split("FORMAT")[0]
117
- result = await local_client.query(f"{sql} FORMAT JSON")
118
- data = result.get("data", [])
125
+ query_result = await local_client.query(f"{sql} FORMAT JSON")
126
+ data = query_result.get("data", [])
119
127
  fixture_name = build_fixture_name(
120
128
  datasource_path.absolute().as_posix(), datasource_name, datasource_content
121
129
  )
@@ -123,9 +131,9 @@ async def create(
123
131
  persist_fixture(fixture_name, data, folder)
124
132
  click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
125
133
 
126
- if cursor and not already_has_cursorrules(folder):
134
+ if cursor:
127
135
  click.echo(FeedbackManager.highlight(message="\n» Creating .cursorrules..."))
128
- create_cursorrules(folder)
136
+ create_rules(folder, source, "cursor")
129
137
  click.echo(FeedbackManager.success(message="✓ Done!\n"))
130
138
 
131
139
  except Exception as e:
@@ -139,9 +147,10 @@ def validate_project_structure(folder: str) -> bool:
139
147
  return all((Path(folder) / path).exists() for path in PROJECT_PATHS)
140
148
 
141
149
 
142
- def validate_fixtures(folder: str) -> bool:
143
- datasource_files = [f for f in os.listdir(Path(folder) / "datasources") if f.endswith(".datasource")]
144
- return len(datasource_files) > 0
150
+ def should_generate_fixtures(result: str) -> bool:
151
+ if not result:
152
+ return False
153
+ return "<type>datasource</type>" in result
145
154
 
146
155
 
147
156
  def already_has_cicd(folder: str) -> bool:
@@ -168,12 +177,12 @@ async def create_resources(
168
177
  prompt: Optional[str],
169
178
  folder: str,
170
179
  ):
171
- force = True
180
+ result = ""
172
181
  folder_path = Path(folder)
173
182
  if data:
174
183
  path = folder_path / data
175
184
  format = path.suffix.lstrip(".")
176
- await _generate_datafile(str(path), local_client, format=format, force=force)
185
+ await _generate_datafile(str(path), local_client, format=format, force=True)
177
186
  name = data.split(".")[0]
178
187
  generate_pipe_file(
179
188
  f"{name}_endpoint",
@@ -185,7 +194,10 @@ TYPE ENDPOINT
185
194
  """,
186
195
  folder,
187
196
  )
188
- return True
197
+ result = (
198
+ f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
199
+ )
200
+
189
201
  elif prompt and user_token:
190
202
  datasource_paths = [
191
203
  Path(folder) / "datasources" / f
@@ -212,41 +224,74 @@ TYPE ENDPOINT
212
224
  ]
213
225
  )
214
226
  llm = LLM(user_token=user_token, host=tb_client.host)
215
- result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
216
- result = extract_xml(result, "response")
217
- resources = parse_xml(result, "resource")
218
- datasources = []
219
- pipes = []
220
- for resource_xml in resources:
221
- resource_type = extract_xml(resource_xml, "type")
222
- name = extract_xml(resource_xml, "name")
223
- content = extract_xml(resource_xml, "content")
224
- resource = {
225
- "name": name,
226
- "content": content,
227
- }
228
- if resource_type.lower() == "datasource":
229
- datasources.append(resource)
230
- elif resource_type.lower() == "pipe":
231
- pipes.append(resource)
232
-
233
- for ds in datasources:
234
- content = ds["content"].replace("```", "")
235
- filename = f"{ds['name']}.datasource"
236
- generate_datafile(
237
- content,
238
- filename=filename,
239
- data=None,
240
- _format="ndjson",
241
- force=force,
242
- folder=folder,
243
- )
244
-
245
- for pipe in pipes:
246
- content = pipe["content"].replace("```", "")
247
- generate_pipe_file(pipe["name"], content, folder)
248
-
249
- return len(datasources) > 0
227
+ result = ""
228
+ iterations = 0
229
+ history = ""
230
+ generated_paths: list[Path] = []
231
+
232
+ while iterations < 10:
233
+ feedback = ""
234
+ if iterations > 0:
235
+ feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
236
+ if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
237
+ break
238
+ else:
239
+ if iterations > 0:
240
+ click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
241
+ for path in generated_paths:
242
+ path.unlink()
243
+ generated_paths = []
244
+
245
+ save_context(prompt, feedback)
246
+ result = llm.ask(system_prompt=create_prompt(resources_xml, feedback, history), prompt=prompt)
247
+ result = extract_xml(result, "response")
248
+ history = (
249
+ history
250
+ + f"""
251
+ <result_iteration_{iterations}>
252
+ {result}
253
+ </result_iteration_{iterations}>
254
+ """
255
+ )
256
+ resources = parse_xml(result, "resource")
257
+ datasources = []
258
+ pipes = []
259
+ for resource_xml in resources:
260
+ resource_type = extract_xml(resource_xml, "type")
261
+ name = extract_xml(resource_xml, "name")
262
+ content = extract_xml(resource_xml, "content")
263
+ resource = {
264
+ "name": name,
265
+ "content": content,
266
+ }
267
+ if resource_type.lower() == "datasource":
268
+ datasources.append(resource)
269
+ elif resource_type.lower() == "pipe":
270
+ pipes.append(resource)
271
+
272
+ for ds in datasources:
273
+ content = ds["content"].replace("```", "")
274
+ filename = f"{ds['name']}.datasource"
275
+ datasource_path = generate_datafile(
276
+ content,
277
+ filename=filename,
278
+ data=None,
279
+ _format="ndjson",
280
+ force=True,
281
+ folder=folder,
282
+ )
283
+ generated_paths.append(datasource_path)
284
+ for pipe in pipes:
285
+ content = pipe["content"].replace("```", "")
286
+ pipe_path = generate_pipe_file(pipe["name"], content, folder)
287
+ generated_paths.append(pipe_path)
288
+
289
+ iterations += 1
290
+
291
+ if iterations == 10:
292
+ click.echo(FeedbackManager.info(message="Too many iterations. Change the prompt and try again."))
293
+
294
+ return result
250
295
 
251
296
 
252
297
  def init_git(folder: str):
@@ -266,7 +311,7 @@ def init_git(folder: str):
266
311
  raise CLIException(f"Error initializing Git: {e}")
267
312
 
268
313
 
269
- def generate_pipe_file(name: str, content: str, folder: str):
314
+ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
270
315
  def is_copy(content: str) -> bool:
271
316
  return re.search(r"TYPE copy", content, re.IGNORECASE) is not None
272
317
 
@@ -292,12 +337,42 @@ def generate_pipe_file(name: str, content: str, folder: str):
292
337
  with open(f"{f}", "w") as file:
293
338
  file.write(content)
294
339
  click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
340
+ return f.relative_to(folder)
341
+
342
+
343
+ def create_rules(folder: str, source: str, agent: str):
344
+ if agent == "cursor":
345
+ extension = ".cursorrules"
346
+ elif agent == "windsurf":
347
+ extension = ".windsurfrules"
348
+ else:
349
+ extension = ".txt"
350
+ rules_file = Path(folder) / extension
351
+ rules_file.write_text(rules_prompt(source))
352
+
353
+
354
+ def get_context_file() -> Path:
355
+ context_file = Path(os.path.expanduser("~/.tb_create_context"))
356
+ if not context_file.exists():
357
+ context_file.touch()
358
+ return context_file
359
+
360
+
361
+ def get_context() -> str:
362
+ context_file = get_context_file()
363
+ return context_file.read_text()
295
364
 
296
365
 
297
- def already_has_cursorrules(folder: str) -> bool:
298
- return (Path(folder) / ".cursorrules").exists()
366
+ def save_context(prompt: str, feedback: str):
367
+ context_file = get_context_file()
368
+ context_file.write_text(f"- {prompt}\n{feedback}")
299
369
 
300
370
 
301
- def create_cursorrules(folder: str):
302
- cursorrules_file = Path(folder) / ".cursorrules"
303
- cursorrules_file.write_text(cursorrules_prompt)
371
+ @cli.command("rules")
372
+ @click.option("--agent", type=str, default="cursor", help="Agent to use for rules")
373
+ @click.option("--source", type=str, default="tb", help="Source of the command")
374
+ @click.pass_context
375
+ def rules(ctx: click.Context, agent: str, source: str):
376
+ """Create agent rules for the project."""
377
+ project: Project = ctx.ensure_object(dict)["project"]
378
+ create_rules(str(project.path), source, agent)
@@ -15,14 +15,12 @@ from tinybird.client import TinyB
15
15
  from tinybird.sql import parse_table_structure, schema_to_sql_columns
16
16
  from tinybird.sql_template import get_used_tables_in_template, render_sql_template
17
17
  from tinybird.tb.modules.common import get_ca_pem_content
18
- from tinybird.tb.modules.datafile.build_common import update_tags_in_resource
19
- from tinybird.tb.modules.datafile.build_datasource import is_datasource, new_ds
18
+ from tinybird.tb.modules.datafile.build_datasource import is_datasource
20
19
  from tinybird.tb.modules.datafile.build_pipe import (
21
20
  get_target_materialized_data_source_name,
22
21
  is_endpoint,
23
22
  is_endpoint_with_no_dependencies,
24
23
  is_materialized,
25
- new_pipe,
26
24
  )
27
25
  from tinybird.tb.modules.datafile.common import (
28
26
  DEFAULT_CRON_PERIOD,
@@ -44,95 +42,32 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
44
42
  from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
45
43
  from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
46
44
  from tinybird.tb.modules.feedback_manager import FeedbackManager
47
- from tinybird.tb.modules.local_common import get_tinybird_local_config
45
+ from tinybird.tb.modules.local_common import get_tinybird_local_client
46
+ from tinybird.tb.modules.project import Project
48
47
 
49
48
 
50
49
  async def folder_build(
51
- tb_client: TinyB,
50
+ project: Project,
52
51
  filenames: Optional[List[str]] = None,
53
- folder: str = ".",
54
- ignore_sql_errors: bool = False,
55
52
  is_internal: bool = False,
56
53
  is_vendor: bool = False,
57
54
  current_ws: Optional[Dict[str, Any]] = None,
58
55
  local_ws: Optional[Dict[str, Any]] = None,
59
56
  watch: bool = False,
60
57
  ):
61
- config = await get_tinybird_local_config(folder)
62
58
  build = True
63
59
  dry_run = False
64
60
  force = True
65
61
  only_changes = True
66
62
  debug = False
67
- check = True
68
- populate = False
69
- populate_subset = None
70
- populate_condition = None
71
- tests_to_run = 0
72
- override_datasource = False
73
- skip_confirmation = True
74
- wait = False
75
- unlink_on_populate_error = False
76
- only_response_times = False
77
63
  run_tests = False
78
64
  verbose = False
79
- as_standard = False
80
65
  raise_on_exists = False
81
66
  fork_downstream = True
82
67
  fork = False
83
68
  release_created = False
84
- tests_relative_change = 0.01
85
- tests_sample_by_params = 0
86
- tests_filter_by = None
87
- tests_failfast = False
88
- tests_ignore_order = False
89
- tests_validate_processed_bytes = False
90
- tests_check_requests_from_branch = False
91
- vendor_paths = []
92
-
93
- vendor_path = Path("vendor")
94
- user_token = config.get_user_token()
95
- user_client = deepcopy(tb_client)
96
-
97
- if user_token:
98
- user_client.token = user_token
99
-
100
- vendor_workspaces = []
101
-
102
- if vendor_path.exists() and not is_vendor and not watch:
103
- user_workspaces = await user_client.user_workspaces()
104
- for x in vendor_path.iterdir():
105
- if x.is_dir() and x.name:
106
- if user_token:
107
- try:
108
- ws_to_delete = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == x.name), None)
109
- if ws_to_delete:
110
- await user_client.delete_workspace(ws_to_delete["id"], hard_delete_confirmation=x.name)
111
- except Exception:
112
- pass
113
- vendor_ws = await user_client.create_workspace(x.name, template=None)
114
- vendor_workspaces.append(vendor_ws)
115
- vendor_paths.append((x.name, str(x)))
116
-
117
- workspaces: List[Dict[str, Any]] = (await user_client.user_workspaces()).get("workspaces", [])
118
-
119
- if not is_vendor:
120
- local_workspace = await tb_client.workspace_info()
121
- local_ws_id = local_workspace.get("id")
122
- local_ws = next((ws for ws in workspaces if ws["id"] == local_ws_id), {})
123
-
124
- current_ws = current_ws or local_ws
125
-
126
- for vendor_ws in [ws for ws in workspaces if ws["name"] in [ws["name"] for ws in vendor_workspaces]]:
127
- ws_client = deepcopy(tb_client)
128
- ws_client.token = vendor_ws["token"]
129
- shared_ws_path = Path(folder) / "vendor" / vendor_ws["name"]
130
-
131
- if shared_ws_path.exists() and not is_vendor and not watch:
132
- await folder_build(
133
- ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws, local_ws=local_ws
134
- )
135
-
69
+ folder = str(project.path)
70
+ tb_client = await get_tinybird_local_client(folder)
136
71
  datasources: List[Dict[str, Any]] = await tb_client.datasources()
137
72
  pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
138
73
 
@@ -150,7 +85,7 @@ async def folder_build(
150
85
  dir_path=folder,
151
86
  process_dependencies=True,
152
87
  skip_connectors=True,
153
- vendor_paths=vendor_paths,
88
+ vendor_paths=[],
154
89
  current_ws=current_ws,
155
90
  only_changes=only_changes,
156
91
  fork_downstream=fork_downstream,
@@ -191,63 +126,14 @@ async def folder_build(
191
126
  resource = to_run[name]["resource"]
192
127
  if not dry_run:
193
128
  if should_push_file(name, remote_resource_names, force, run_tests):
194
- click.echo(FeedbackManager.info_processing_new_resource(name=name, version=""))
195
- try:
196
- await exec_file(
197
- to_run[name],
198
- tb_client,
199
- force,
200
- check,
201
- debug and verbose,
202
- populate,
203
- populate_subset,
204
- populate_condition,
205
- unlink_on_populate_error,
206
- wait,
207
- user_token,
208
- override_datasource,
209
- ignore_sql_errors,
210
- skip_confirmation,
211
- only_response_times,
212
- run_tests,
213
- as_standard,
214
- tests_to_run,
215
- tests_relative_change,
216
- tests_sample_by_params,
217
- tests_filter_by,
218
- tests_failfast,
219
- tests_ignore_order,
220
- tests_validate_processed_bytes,
221
- tests_check_requests_from_branch,
222
- current_ws,
223
- local_ws,
224
- fork_downstream,
225
- fork,
226
- build,
227
- is_vendor,
228
- )
229
- if not run_tests:
230
- click.echo(
231
- FeedbackManager.success_create(
232
- name=(
233
- name
234
- if to_run[name]["version"] is None
235
- else f'{name}__v{to_run[name]["version"]}'
236
- )
237
- )
238
- )
239
- except Exception as e:
240
- filename = to_run[name]["filename"]
241
- exception = FeedbackManager.error_push_file_exception(
242
- filename=filename,
243
- error=e,
244
- )
245
- raise click.ClickException(exception)
129
+ filename = to_run[name]["filename"]
130
+ filename = filename.replace(f"{folder}/", "")
131
+ click.echo(FeedbackManager.info(message=f"✓ {filename}"))
246
132
  else:
247
133
  if raise_on_exists:
248
134
  raise AlreadyExistsException(
249
135
  FeedbackManager.warning_name_already_exists(
250
- name=name if to_run[name]["version"] is None else f'{name}__v{to_run[name]["version"]}'
136
+ name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
251
137
  )
252
138
  )
253
139
  else:
@@ -262,7 +148,7 @@ async def folder_build(
262
148
  name=(
263
149
  name
264
150
  if to_run[name]["version"] is None
265
- else f'{name}__v{to_run[name]["version"]}'
151
+ else f"{name}__v{to_run[name]['version']}"
266
152
  )
267
153
  )
268
154
  )
@@ -304,19 +190,6 @@ async def folder_build(
304
190
  # We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
305
191
  groups = [group for group in toposort(dependencies_graph_fork_downstream)]
306
192
 
307
- for group in groups:
308
- for name in group:
309
- is_vendor = resources_to_run_fork_downstream.get(name, {}).get("filename", "").startswith("vendor/")
310
- if not is_vendor:
311
- try:
312
- await tb_client.datasource_delete(name, force=True)
313
- except Exception:
314
- pass
315
- try:
316
- await tb_client.pipe_delete(name)
317
- except Exception:
318
- pass
319
-
320
193
  groups.reverse()
321
194
  for group in groups:
322
195
  for name in group:
@@ -433,88 +306,6 @@ async def name_matches_existing_resource(resource: str, name: str, tb_client: Ti
433
306
  return False
434
307
 
435
308
 
436
- async def exec_file(
437
- r: Dict[str, Any],
438
- tb_client: TinyB,
439
- force: bool,
440
- check: bool,
441
- debug: bool,
442
- populate: bool,
443
- populate_subset,
444
- populate_condition,
445
- unlink_on_populate_error,
446
- wait_populate,
447
- user_token: Optional[str],
448
- override_datasource: bool = False,
449
- ignore_sql_errors: bool = False,
450
- skip_confirmation: bool = False,
451
- only_response_times: bool = False,
452
- run_tests=False,
453
- as_standard=False,
454
- tests_to_run: int = 0,
455
- tests_relative_change: float = 0.01,
456
- tests_to_sample_by_params: int = 0,
457
- tests_filter_by: Optional[List[str]] = None,
458
- tests_failfast: bool = False,
459
- tests_ignore_order: bool = False,
460
- tests_validate_processed_bytes: bool = False,
461
- tests_check_requests_from_branch: bool = False,
462
- current_ws: Optional[Dict[str, Any]] = None,
463
- local_ws: Optional[Dict[str, Any]] = None,
464
- fork_downstream: Optional[bool] = False,
465
- fork: Optional[bool] = False,
466
- build: Optional[bool] = False,
467
- is_vendor: Optional[bool] = False,
468
- ):
469
- if debug:
470
- click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
471
- if r["resource"] == "pipes":
472
- await new_pipe(
473
- r,
474
- tb_client,
475
- force,
476
- check,
477
- populate,
478
- populate_subset,
479
- populate_condition,
480
- unlink_on_populate_error,
481
- wait_populate,
482
- ignore_sql_errors=ignore_sql_errors,
483
- only_response_times=only_response_times,
484
- run_tests=run_tests,
485
- as_standard=as_standard,
486
- tests_to_run=tests_to_run,
487
- tests_relative_change=tests_relative_change,
488
- tests_to_sample_by_params=tests_to_sample_by_params,
489
- tests_filter_by=tests_filter_by,
490
- tests_failfast=tests_failfast,
491
- tests_ignore_order=tests_ignore_order,
492
- tests_validate_processed_bytes=tests_validate_processed_bytes,
493
- override_datasource=override_datasource,
494
- tests_check_requests_from_branch=tests_check_requests_from_branch,
495
- fork_downstream=fork_downstream,
496
- fork=fork,
497
- )
498
- await update_tags_in_resource(r, "pipe", tb_client)
499
- elif r["resource"] == "datasources":
500
- await new_ds(
501
- r,
502
- tb_client,
503
- user_token,
504
- force,
505
- skip_confirmation=skip_confirmation,
506
- current_ws=current_ws,
507
- local_ws=local_ws,
508
- fork_downstream=fork_downstream,
509
- fork=fork,
510
- build=build,
511
- is_vendor=is_vendor,
512
- )
513
- await update_tags_in_resource(r, "datasource", tb_client)
514
- else:
515
- raise click.ClickException(FeedbackManager.error_unknown_resource(resource=r["resource"]))
516
-
517
-
518
309
  def get_remote_resource_name_without_version(remote_resource_name: str) -> str:
519
310
  """
520
311
  >>> get_remote_resource_name_without_version("r__datasource")
@@ -526,7 +526,7 @@ def format_parse_error(
526
526
  message += f" found at position {adjusted_position - len(keyword)}"
527
527
  else:
528
528
  message += (
529
- f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
529
+ f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
530
530
  )
531
531
  return message
532
532
 
@@ -148,7 +148,7 @@ async def format_engine(
148
148
  else:
149
149
  if node.get("engine", None):
150
150
  empty = '""'
151
- file_parts.append(f'ENGINE {node["engine"]["type"]}' if node.get("engine", {}).get("type") else empty)
151
+ file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
152
152
  file_parts.append(DATAFILE_NEW_LINE)
153
153
  for arg in sorted(node["engine"].get("args", [])):
154
154
  elem = ", ".join([x.strip() for x in arg[1].split(",")])
@@ -42,7 +42,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
42
42
  if node_type == PipeNodeTypes.MATERIALIZED:
43
43
  file_parts.append(node_type_upper)
44
44
  file_parts.append(DATAFILE_NEW_LINE)
45
- file_parts.append(f'DATASOURCE {node["datasource"]}')
45
+ file_parts.append(f"DATASOURCE {node['datasource']}")
46
46
  file_parts.append(DATAFILE_NEW_LINE)
47
47
  await format_engine(file_parts, node)
48
48
 
@@ -50,10 +50,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
50
50
  if node_type == PipeNodeTypes.COPY:
51
51
  file_parts.append(node_type_upper)
52
52
  file_parts.append(DATAFILE_NEW_LINE)
53
- file_parts.append(f'TARGET_DATASOURCE {node["target_datasource"]}')
53
+ file_parts.append(f"TARGET_DATASOURCE {node['target_datasource']}")
54
54
  if node.get("mode"):
55
55
  file_parts.append(DATAFILE_NEW_LINE)
56
- file_parts.append(f'COPY_MODE {node.get("mode")}')
56
+ file_parts.append(f"COPY_MODE {node.get('mode')}")
57
57
 
58
58
  if node.get(CopyParameters.COPY_SCHEDULE):
59
59
  is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
@@ -112,7 +112,7 @@ async def format_node(
112
112
  if item and not unroll_includes:
113
113
  return
114
114
 
115
- file_parts.append(f'NODE {node["name"].strip()}')
115
+ file_parts.append(f"NODE {node['name'].strip()}")
116
116
  file_parts.append(DATAFILE_NEW_LINE)
117
117
 
118
118
  from collections import namedtuple