tinybird 0.0.1.dev43__py3-none-any.whl → 0.0.1.dev46__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/client.py +17 -1
- tinybird/prompts.py +135 -15
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -1
- tinybird/tb/modules/build.py +28 -20
- tinybird/tb/modules/cli.py +18 -62
- tinybird/tb/modules/common.py +3 -2
- tinybird/tb/modules/copy.py +1 -1
- tinybird/tb/modules/create.py +134 -59
- tinybird/tb/modules/datafile/build.py +12 -221
- tinybird/tb/modules/datafile/common.py +1 -1
- tinybird/tb/modules/datafile/format_datasource.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +4 -4
- tinybird/tb/modules/datafile/pipe_checker.py +3 -3
- tinybird/tb/modules/datasource.py +1 -1
- tinybird/tb/modules/deployment.py +1 -1
- tinybird/tb/modules/endpoint.py +89 -2
- tinybird/tb/modules/feedback_manager.py +5 -1
- tinybird/tb/modules/local_common.py +10 -7
- tinybird/tb/modules/materialization.py +146 -0
- tinybird/tb/modules/mock.py +56 -16
- tinybird/tb/modules/pipe.py +8 -326
- tinybird/tb/modules/project.py +10 -4
- tinybird/tb/modules/shell.py +3 -3
- tinybird/tb/modules/test.py +73 -38
- tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
- tinybird/tb/modules/update.py +1 -1
- tinybird/tb/modules/workspace.py +2 -1
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/RECORD +33 -33
- tinybird/tb/modules/build_client.py +0 -199
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev46.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/create.py
CHANGED
|
@@ -7,7 +7,7 @@ from typing import Optional
|
|
|
7
7
|
import click
|
|
8
8
|
|
|
9
9
|
from tinybird.client import TinyB
|
|
10
|
-
from tinybird.prompts import create_prompt,
|
|
10
|
+
from tinybird.prompts import create_prompt, mock_prompt, rules_prompt
|
|
11
11
|
from tinybird.tb.modules.cicd import init_cicd
|
|
12
12
|
from tinybird.tb.modules.cli import cli
|
|
13
13
|
from tinybird.tb.modules.common import _generate_datafile, check_user_token_with_client, coro, generate_datafile
|
|
@@ -18,6 +18,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
|
18
18
|
from tinybird.tb.modules.llm import LLM
|
|
19
19
|
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
20
20
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
21
|
+
from tinybird.tb.modules.project import Project
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
@cli.command()
|
|
@@ -35,28 +36,33 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
|
35
36
|
)
|
|
36
37
|
@click.option(
|
|
37
38
|
"--folder",
|
|
38
|
-
default="
|
|
39
|
+
default="",
|
|
39
40
|
type=click.Path(exists=False, file_okay=False),
|
|
40
41
|
help="Folder where datafiles will be placed",
|
|
41
42
|
)
|
|
42
43
|
@click.option("--rows", type=int, default=10, help="Number of events to send")
|
|
43
44
|
@click.option("--cursor", default=False, is_flag=True, help="Create .cursorrules file with Tinybird rules")
|
|
45
|
+
@click.option("--source", type=str, default="tb", help="Source of the command")
|
|
46
|
+
@click.pass_context
|
|
44
47
|
@coro
|
|
45
48
|
async def create(
|
|
49
|
+
ctx: click.Context,
|
|
46
50
|
data: Optional[str],
|
|
47
51
|
prompt: Optional[str],
|
|
48
52
|
folder: Optional[str],
|
|
49
53
|
rows: int,
|
|
50
54
|
cursor: bool,
|
|
55
|
+
source: str,
|
|
51
56
|
) -> None:
|
|
52
57
|
"""Initialize a new project."""
|
|
58
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
53
59
|
folder = folder or getcwd()
|
|
54
60
|
folder_path = Path(folder)
|
|
55
61
|
if not folder_path.exists():
|
|
56
62
|
folder_path.mkdir()
|
|
57
63
|
|
|
58
64
|
try:
|
|
59
|
-
config = CLIConfig.get_project_config(
|
|
65
|
+
config = CLIConfig.get_project_config(str(project.path))
|
|
60
66
|
tb_client = config.get_client()
|
|
61
67
|
user_token: Optional[str] = None
|
|
62
68
|
if prompt:
|
|
@@ -79,9 +85,11 @@ async def create(
|
|
|
79
85
|
create_project_structure(folder)
|
|
80
86
|
click.echo(FeedbackManager.success(message="✓ Scaffolding completed!\n"))
|
|
81
87
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
88
|
+
result = ""
|
|
89
|
+
if data or prompt:
|
|
90
|
+
click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
|
|
91
|
+
result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
|
|
92
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
85
93
|
|
|
86
94
|
if not already_has_cicd(folder):
|
|
87
95
|
click.echo(FeedbackManager.highlight(message="\n» Creating CI/CD files for GitHub and GitLab..."))
|
|
@@ -89,7 +97,7 @@ async def create(
|
|
|
89
97
|
await init_cicd(data_project_dir=os.path.relpath(folder))
|
|
90
98
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
91
99
|
|
|
92
|
-
if
|
|
100
|
+
if should_generate_fixtures(result):
|
|
93
101
|
click.echo(FeedbackManager.highlight(message="\n» Generating fixtures..."))
|
|
94
102
|
|
|
95
103
|
if data:
|
|
@@ -110,12 +118,12 @@ async def create(
|
|
|
110
118
|
datasource_content = datasource_path.read_text()
|
|
111
119
|
has_json_path = "`json:" in datasource_content
|
|
112
120
|
if has_json_path:
|
|
113
|
-
prompt = f"<datasource_schema>{datasource_content}</datasource_schema
|
|
121
|
+
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>"
|
|
114
122
|
response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
|
|
115
123
|
sql = extract_xml(response, "sql")
|
|
116
124
|
sql = sql.split("FORMAT")[0]
|
|
117
|
-
|
|
118
|
-
data =
|
|
125
|
+
query_result = await local_client.query(f"{sql} FORMAT JSON")
|
|
126
|
+
data = query_result.get("data", [])
|
|
119
127
|
fixture_name = build_fixture_name(
|
|
120
128
|
datasource_path.absolute().as_posix(), datasource_name, datasource_content
|
|
121
129
|
)
|
|
@@ -123,9 +131,9 @@ async def create(
|
|
|
123
131
|
persist_fixture(fixture_name, data, folder)
|
|
124
132
|
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}"))
|
|
125
133
|
|
|
126
|
-
if cursor
|
|
134
|
+
if cursor:
|
|
127
135
|
click.echo(FeedbackManager.highlight(message="\n» Creating .cursorrules..."))
|
|
128
|
-
|
|
136
|
+
create_rules(folder, source, "cursor")
|
|
129
137
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
130
138
|
|
|
131
139
|
except Exception as e:
|
|
@@ -139,9 +147,10 @@ def validate_project_structure(folder: str) -> bool:
|
|
|
139
147
|
return all((Path(folder) / path).exists() for path in PROJECT_PATHS)
|
|
140
148
|
|
|
141
149
|
|
|
142
|
-
def
|
|
143
|
-
|
|
144
|
-
|
|
150
|
+
def should_generate_fixtures(result: str) -> bool:
|
|
151
|
+
if not result:
|
|
152
|
+
return False
|
|
153
|
+
return "<type>datasource</type>" in result
|
|
145
154
|
|
|
146
155
|
|
|
147
156
|
def already_has_cicd(folder: str) -> bool:
|
|
@@ -168,12 +177,12 @@ async def create_resources(
|
|
|
168
177
|
prompt: Optional[str],
|
|
169
178
|
folder: str,
|
|
170
179
|
):
|
|
171
|
-
|
|
180
|
+
result = ""
|
|
172
181
|
folder_path = Path(folder)
|
|
173
182
|
if data:
|
|
174
183
|
path = folder_path / data
|
|
175
184
|
format = path.suffix.lstrip(".")
|
|
176
|
-
await _generate_datafile(str(path), local_client, format=format, force=
|
|
185
|
+
await _generate_datafile(str(path), local_client, format=format, force=True)
|
|
177
186
|
name = data.split(".")[0]
|
|
178
187
|
generate_pipe_file(
|
|
179
188
|
f"{name}_endpoint",
|
|
@@ -185,7 +194,10 @@ TYPE ENDPOINT
|
|
|
185
194
|
""",
|
|
186
195
|
folder,
|
|
187
196
|
)
|
|
188
|
-
|
|
197
|
+
result = (
|
|
198
|
+
f"<response><resource><type>datasource</type><name>{name}</name><content></content></resource></response>"
|
|
199
|
+
)
|
|
200
|
+
|
|
189
201
|
elif prompt and user_token:
|
|
190
202
|
datasource_paths = [
|
|
191
203
|
Path(folder) / "datasources" / f
|
|
@@ -212,41 +224,74 @@ TYPE ENDPOINT
|
|
|
212
224
|
]
|
|
213
225
|
)
|
|
214
226
|
llm = LLM(user_token=user_token, host=tb_client.host)
|
|
215
|
-
result =
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
227
|
+
result = ""
|
|
228
|
+
iterations = 0
|
|
229
|
+
history = ""
|
|
230
|
+
generated_paths: list[Path] = []
|
|
231
|
+
|
|
232
|
+
while iterations < 10:
|
|
233
|
+
feedback = ""
|
|
234
|
+
if iterations > 0:
|
|
235
|
+
feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
|
|
236
|
+
if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
|
|
237
|
+
break
|
|
238
|
+
else:
|
|
239
|
+
if iterations > 0:
|
|
240
|
+
click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
|
|
241
|
+
for path in generated_paths:
|
|
242
|
+
path.unlink()
|
|
243
|
+
generated_paths = []
|
|
244
|
+
|
|
245
|
+
save_context(prompt, feedback)
|
|
246
|
+
result = llm.ask(system_prompt=create_prompt(resources_xml, feedback, history), prompt=prompt)
|
|
247
|
+
result = extract_xml(result, "response")
|
|
248
|
+
history = (
|
|
249
|
+
history
|
|
250
|
+
+ f"""
|
|
251
|
+
<result_iteration_{iterations}>
|
|
252
|
+
{result}
|
|
253
|
+
</result_iteration_{iterations}>
|
|
254
|
+
"""
|
|
255
|
+
)
|
|
256
|
+
resources = parse_xml(result, "resource")
|
|
257
|
+
datasources = []
|
|
258
|
+
pipes = []
|
|
259
|
+
for resource_xml in resources:
|
|
260
|
+
resource_type = extract_xml(resource_xml, "type")
|
|
261
|
+
name = extract_xml(resource_xml, "name")
|
|
262
|
+
content = extract_xml(resource_xml, "content")
|
|
263
|
+
resource = {
|
|
264
|
+
"name": name,
|
|
265
|
+
"content": content,
|
|
266
|
+
}
|
|
267
|
+
if resource_type.lower() == "datasource":
|
|
268
|
+
datasources.append(resource)
|
|
269
|
+
elif resource_type.lower() == "pipe":
|
|
270
|
+
pipes.append(resource)
|
|
271
|
+
|
|
272
|
+
for ds in datasources:
|
|
273
|
+
content = ds["content"].replace("```", "")
|
|
274
|
+
filename = f"{ds['name']}.datasource"
|
|
275
|
+
datasource_path = generate_datafile(
|
|
276
|
+
content,
|
|
277
|
+
filename=filename,
|
|
278
|
+
data=None,
|
|
279
|
+
_format="ndjson",
|
|
280
|
+
force=True,
|
|
281
|
+
folder=folder,
|
|
282
|
+
)
|
|
283
|
+
generated_paths.append(datasource_path)
|
|
284
|
+
for pipe in pipes:
|
|
285
|
+
content = pipe["content"].replace("```", "")
|
|
286
|
+
pipe_path = generate_pipe_file(pipe["name"], content, folder)
|
|
287
|
+
generated_paths.append(pipe_path)
|
|
288
|
+
|
|
289
|
+
iterations += 1
|
|
290
|
+
|
|
291
|
+
if iterations == 10:
|
|
292
|
+
click.echo(FeedbackManager.info(message="Too many iterations. Change the prompt and try again."))
|
|
293
|
+
|
|
294
|
+
return result
|
|
250
295
|
|
|
251
296
|
|
|
252
297
|
def init_git(folder: str):
|
|
@@ -266,7 +311,7 @@ def init_git(folder: str):
|
|
|
266
311
|
raise CLIException(f"Error initializing Git: {e}")
|
|
267
312
|
|
|
268
313
|
|
|
269
|
-
def generate_pipe_file(name: str, content: str, folder: str):
|
|
314
|
+
def generate_pipe_file(name: str, content: str, folder: str) -> Path:
|
|
270
315
|
def is_copy(content: str) -> bool:
|
|
271
316
|
return re.search(r"TYPE copy", content, re.IGNORECASE) is not None
|
|
272
317
|
|
|
@@ -292,12 +337,42 @@ def generate_pipe_file(name: str, content: str, folder: str):
|
|
|
292
337
|
with open(f"{f}", "w") as file:
|
|
293
338
|
file.write(content)
|
|
294
339
|
click.echo(FeedbackManager.info_file_created(file=f.relative_to(folder)))
|
|
340
|
+
return f.relative_to(folder)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def create_rules(folder: str, source: str, agent: str):
|
|
344
|
+
if agent == "cursor":
|
|
345
|
+
extension = ".cursorrules"
|
|
346
|
+
elif agent == "windsurf":
|
|
347
|
+
extension = ".windsurfrules"
|
|
348
|
+
else:
|
|
349
|
+
extension = ".txt"
|
|
350
|
+
rules_file = Path(folder) / extension
|
|
351
|
+
rules_file.write_text(rules_prompt(source))
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def get_context_file() -> Path:
|
|
355
|
+
context_file = Path(os.path.expanduser("~/.tb_create_context"))
|
|
356
|
+
if not context_file.exists():
|
|
357
|
+
context_file.touch()
|
|
358
|
+
return context_file
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def get_context() -> str:
|
|
362
|
+
context_file = get_context_file()
|
|
363
|
+
return context_file.read_text()
|
|
295
364
|
|
|
296
365
|
|
|
297
|
-
def
|
|
298
|
-
|
|
366
|
+
def save_context(prompt: str, feedback: str):
|
|
367
|
+
context_file = get_context_file()
|
|
368
|
+
context_file.write_text(f"- {prompt}\n{feedback}")
|
|
299
369
|
|
|
300
370
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
371
|
+
@cli.command("rules")
|
|
372
|
+
@click.option("--agent", type=str, default="cursor", help="Agent to use for rules")
|
|
373
|
+
@click.option("--source", type=str, default="tb", help="Source of the command")
|
|
374
|
+
@click.pass_context
|
|
375
|
+
def rules(ctx: click.Context, agent: str, source: str):
|
|
376
|
+
"""Create agent rules for the project."""
|
|
377
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
378
|
+
create_rules(str(project.path), source, agent)
|
|
@@ -15,14 +15,12 @@ from tinybird.client import TinyB
|
|
|
15
15
|
from tinybird.sql import parse_table_structure, schema_to_sql_columns
|
|
16
16
|
from tinybird.sql_template import get_used_tables_in_template, render_sql_template
|
|
17
17
|
from tinybird.tb.modules.common import get_ca_pem_content
|
|
18
|
-
from tinybird.tb.modules.datafile.
|
|
19
|
-
from tinybird.tb.modules.datafile.build_datasource import is_datasource, new_ds
|
|
18
|
+
from tinybird.tb.modules.datafile.build_datasource import is_datasource
|
|
20
19
|
from tinybird.tb.modules.datafile.build_pipe import (
|
|
21
20
|
get_target_materialized_data_source_name,
|
|
22
21
|
is_endpoint,
|
|
23
22
|
is_endpoint_with_no_dependencies,
|
|
24
23
|
is_materialized,
|
|
25
|
-
new_pipe,
|
|
26
24
|
)
|
|
27
25
|
from tinybird.tb.modules.datafile.common import (
|
|
28
26
|
DEFAULT_CRON_PERIOD,
|
|
@@ -44,95 +42,32 @@ from tinybird.tb.modules.datafile.exceptions import AlreadyExistsException, Incl
|
|
|
44
42
|
from tinybird.tb.modules.datafile.parse_datasource import parse_datasource
|
|
45
43
|
from tinybird.tb.modules.datafile.parse_pipe import parse_pipe
|
|
46
44
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
47
|
-
from tinybird.tb.modules.local_common import
|
|
45
|
+
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
46
|
+
from tinybird.tb.modules.project import Project
|
|
48
47
|
|
|
49
48
|
|
|
50
49
|
async def folder_build(
|
|
51
|
-
|
|
50
|
+
project: Project,
|
|
52
51
|
filenames: Optional[List[str]] = None,
|
|
53
|
-
folder: str = ".",
|
|
54
|
-
ignore_sql_errors: bool = False,
|
|
55
52
|
is_internal: bool = False,
|
|
56
53
|
is_vendor: bool = False,
|
|
57
54
|
current_ws: Optional[Dict[str, Any]] = None,
|
|
58
55
|
local_ws: Optional[Dict[str, Any]] = None,
|
|
59
56
|
watch: bool = False,
|
|
60
57
|
):
|
|
61
|
-
config = await get_tinybird_local_config(folder)
|
|
62
58
|
build = True
|
|
63
59
|
dry_run = False
|
|
64
60
|
force = True
|
|
65
61
|
only_changes = True
|
|
66
62
|
debug = False
|
|
67
|
-
check = True
|
|
68
|
-
populate = False
|
|
69
|
-
populate_subset = None
|
|
70
|
-
populate_condition = None
|
|
71
|
-
tests_to_run = 0
|
|
72
|
-
override_datasource = False
|
|
73
|
-
skip_confirmation = True
|
|
74
|
-
wait = False
|
|
75
|
-
unlink_on_populate_error = False
|
|
76
|
-
only_response_times = False
|
|
77
63
|
run_tests = False
|
|
78
64
|
verbose = False
|
|
79
|
-
as_standard = False
|
|
80
65
|
raise_on_exists = False
|
|
81
66
|
fork_downstream = True
|
|
82
67
|
fork = False
|
|
83
68
|
release_created = False
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
tests_filter_by = None
|
|
87
|
-
tests_failfast = False
|
|
88
|
-
tests_ignore_order = False
|
|
89
|
-
tests_validate_processed_bytes = False
|
|
90
|
-
tests_check_requests_from_branch = False
|
|
91
|
-
vendor_paths = []
|
|
92
|
-
|
|
93
|
-
vendor_path = Path("vendor")
|
|
94
|
-
user_token = config.get_user_token()
|
|
95
|
-
user_client = deepcopy(tb_client)
|
|
96
|
-
|
|
97
|
-
if user_token:
|
|
98
|
-
user_client.token = user_token
|
|
99
|
-
|
|
100
|
-
vendor_workspaces = []
|
|
101
|
-
|
|
102
|
-
if vendor_path.exists() and not is_vendor and not watch:
|
|
103
|
-
user_workspaces = await user_client.user_workspaces()
|
|
104
|
-
for x in vendor_path.iterdir():
|
|
105
|
-
if x.is_dir() and x.name:
|
|
106
|
-
if user_token:
|
|
107
|
-
try:
|
|
108
|
-
ws_to_delete = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == x.name), None)
|
|
109
|
-
if ws_to_delete:
|
|
110
|
-
await user_client.delete_workspace(ws_to_delete["id"], hard_delete_confirmation=x.name)
|
|
111
|
-
except Exception:
|
|
112
|
-
pass
|
|
113
|
-
vendor_ws = await user_client.create_workspace(x.name, template=None)
|
|
114
|
-
vendor_workspaces.append(vendor_ws)
|
|
115
|
-
vendor_paths.append((x.name, str(x)))
|
|
116
|
-
|
|
117
|
-
workspaces: List[Dict[str, Any]] = (await user_client.user_workspaces()).get("workspaces", [])
|
|
118
|
-
|
|
119
|
-
if not is_vendor:
|
|
120
|
-
local_workspace = await tb_client.workspace_info()
|
|
121
|
-
local_ws_id = local_workspace.get("id")
|
|
122
|
-
local_ws = next((ws for ws in workspaces if ws["id"] == local_ws_id), {})
|
|
123
|
-
|
|
124
|
-
current_ws = current_ws or local_ws
|
|
125
|
-
|
|
126
|
-
for vendor_ws in [ws for ws in workspaces if ws["name"] in [ws["name"] for ws in vendor_workspaces]]:
|
|
127
|
-
ws_client = deepcopy(tb_client)
|
|
128
|
-
ws_client.token = vendor_ws["token"]
|
|
129
|
-
shared_ws_path = Path(folder) / "vendor" / vendor_ws["name"]
|
|
130
|
-
|
|
131
|
-
if shared_ws_path.exists() and not is_vendor and not watch:
|
|
132
|
-
await folder_build(
|
|
133
|
-
ws_client, folder=shared_ws_path.as_posix(), is_vendor=True, current_ws=vendor_ws, local_ws=local_ws
|
|
134
|
-
)
|
|
135
|
-
|
|
69
|
+
folder = str(project.path)
|
|
70
|
+
tb_client = await get_tinybird_local_client(folder)
|
|
136
71
|
datasources: List[Dict[str, Any]] = await tb_client.datasources()
|
|
137
72
|
pipes: List[Dict[str, Any]] = await tb_client.pipes(dependencies=True)
|
|
138
73
|
|
|
@@ -150,7 +85,7 @@ async def folder_build(
|
|
|
150
85
|
dir_path=folder,
|
|
151
86
|
process_dependencies=True,
|
|
152
87
|
skip_connectors=True,
|
|
153
|
-
vendor_paths=
|
|
88
|
+
vendor_paths=[],
|
|
154
89
|
current_ws=current_ws,
|
|
155
90
|
only_changes=only_changes,
|
|
156
91
|
fork_downstream=fork_downstream,
|
|
@@ -191,63 +126,14 @@ async def folder_build(
|
|
|
191
126
|
resource = to_run[name]["resource"]
|
|
192
127
|
if not dry_run:
|
|
193
128
|
if should_push_file(name, remote_resource_names, force, run_tests):
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
to_run[name],
|
|
198
|
-
tb_client,
|
|
199
|
-
force,
|
|
200
|
-
check,
|
|
201
|
-
debug and verbose,
|
|
202
|
-
populate,
|
|
203
|
-
populate_subset,
|
|
204
|
-
populate_condition,
|
|
205
|
-
unlink_on_populate_error,
|
|
206
|
-
wait,
|
|
207
|
-
user_token,
|
|
208
|
-
override_datasource,
|
|
209
|
-
ignore_sql_errors,
|
|
210
|
-
skip_confirmation,
|
|
211
|
-
only_response_times,
|
|
212
|
-
run_tests,
|
|
213
|
-
as_standard,
|
|
214
|
-
tests_to_run,
|
|
215
|
-
tests_relative_change,
|
|
216
|
-
tests_sample_by_params,
|
|
217
|
-
tests_filter_by,
|
|
218
|
-
tests_failfast,
|
|
219
|
-
tests_ignore_order,
|
|
220
|
-
tests_validate_processed_bytes,
|
|
221
|
-
tests_check_requests_from_branch,
|
|
222
|
-
current_ws,
|
|
223
|
-
local_ws,
|
|
224
|
-
fork_downstream,
|
|
225
|
-
fork,
|
|
226
|
-
build,
|
|
227
|
-
is_vendor,
|
|
228
|
-
)
|
|
229
|
-
if not run_tests:
|
|
230
|
-
click.echo(
|
|
231
|
-
FeedbackManager.success_create(
|
|
232
|
-
name=(
|
|
233
|
-
name
|
|
234
|
-
if to_run[name]["version"] is None
|
|
235
|
-
else f'{name}__v{to_run[name]["version"]}'
|
|
236
|
-
)
|
|
237
|
-
)
|
|
238
|
-
)
|
|
239
|
-
except Exception as e:
|
|
240
|
-
filename = to_run[name]["filename"]
|
|
241
|
-
exception = FeedbackManager.error_push_file_exception(
|
|
242
|
-
filename=filename,
|
|
243
|
-
error=e,
|
|
244
|
-
)
|
|
245
|
-
raise click.ClickException(exception)
|
|
129
|
+
filename = to_run[name]["filename"]
|
|
130
|
+
filename = filename.replace(f"{folder}/", "")
|
|
131
|
+
click.echo(FeedbackManager.info(message=f"✓ {filename}"))
|
|
246
132
|
else:
|
|
247
133
|
if raise_on_exists:
|
|
248
134
|
raise AlreadyExistsException(
|
|
249
135
|
FeedbackManager.warning_name_already_exists(
|
|
250
|
-
name=name if to_run[name]["version"] is None else f
|
|
136
|
+
name=name if to_run[name]["version"] is None else f"{name}__v{to_run[name]['version']}"
|
|
251
137
|
)
|
|
252
138
|
)
|
|
253
139
|
else:
|
|
@@ -262,7 +148,7 @@ async def folder_build(
|
|
|
262
148
|
name=(
|
|
263
149
|
name
|
|
264
150
|
if to_run[name]["version"] is None
|
|
265
|
-
else f
|
|
151
|
+
else f"{name}__v{to_run[name]['version']}"
|
|
266
152
|
)
|
|
267
153
|
)
|
|
268
154
|
)
|
|
@@ -304,19 +190,6 @@ async def folder_build(
|
|
|
304
190
|
# We need to deploy the datasources from left to right as some datasources might have MV that depend on the column types of previous datasources. Ex: `test_change_column_type_landing_datasource` test
|
|
305
191
|
groups = [group for group in toposort(dependencies_graph_fork_downstream)]
|
|
306
192
|
|
|
307
|
-
for group in groups:
|
|
308
|
-
for name in group:
|
|
309
|
-
is_vendor = resources_to_run_fork_downstream.get(name, {}).get("filename", "").startswith("vendor/")
|
|
310
|
-
if not is_vendor:
|
|
311
|
-
try:
|
|
312
|
-
await tb_client.datasource_delete(name, force=True)
|
|
313
|
-
except Exception:
|
|
314
|
-
pass
|
|
315
|
-
try:
|
|
316
|
-
await tb_client.pipe_delete(name)
|
|
317
|
-
except Exception:
|
|
318
|
-
pass
|
|
319
|
-
|
|
320
193
|
groups.reverse()
|
|
321
194
|
for group in groups:
|
|
322
195
|
for name in group:
|
|
@@ -433,88 +306,6 @@ async def name_matches_existing_resource(resource: str, name: str, tb_client: Ti
|
|
|
433
306
|
return False
|
|
434
307
|
|
|
435
308
|
|
|
436
|
-
async def exec_file(
|
|
437
|
-
r: Dict[str, Any],
|
|
438
|
-
tb_client: TinyB,
|
|
439
|
-
force: bool,
|
|
440
|
-
check: bool,
|
|
441
|
-
debug: bool,
|
|
442
|
-
populate: bool,
|
|
443
|
-
populate_subset,
|
|
444
|
-
populate_condition,
|
|
445
|
-
unlink_on_populate_error,
|
|
446
|
-
wait_populate,
|
|
447
|
-
user_token: Optional[str],
|
|
448
|
-
override_datasource: bool = False,
|
|
449
|
-
ignore_sql_errors: bool = False,
|
|
450
|
-
skip_confirmation: bool = False,
|
|
451
|
-
only_response_times: bool = False,
|
|
452
|
-
run_tests=False,
|
|
453
|
-
as_standard=False,
|
|
454
|
-
tests_to_run: int = 0,
|
|
455
|
-
tests_relative_change: float = 0.01,
|
|
456
|
-
tests_to_sample_by_params: int = 0,
|
|
457
|
-
tests_filter_by: Optional[List[str]] = None,
|
|
458
|
-
tests_failfast: bool = False,
|
|
459
|
-
tests_ignore_order: bool = False,
|
|
460
|
-
tests_validate_processed_bytes: bool = False,
|
|
461
|
-
tests_check_requests_from_branch: bool = False,
|
|
462
|
-
current_ws: Optional[Dict[str, Any]] = None,
|
|
463
|
-
local_ws: Optional[Dict[str, Any]] = None,
|
|
464
|
-
fork_downstream: Optional[bool] = False,
|
|
465
|
-
fork: Optional[bool] = False,
|
|
466
|
-
build: Optional[bool] = False,
|
|
467
|
-
is_vendor: Optional[bool] = False,
|
|
468
|
-
):
|
|
469
|
-
if debug:
|
|
470
|
-
click.echo(FeedbackManager.debug_running_file(file=pp.pformat(r)))
|
|
471
|
-
if r["resource"] == "pipes":
|
|
472
|
-
await new_pipe(
|
|
473
|
-
r,
|
|
474
|
-
tb_client,
|
|
475
|
-
force,
|
|
476
|
-
check,
|
|
477
|
-
populate,
|
|
478
|
-
populate_subset,
|
|
479
|
-
populate_condition,
|
|
480
|
-
unlink_on_populate_error,
|
|
481
|
-
wait_populate,
|
|
482
|
-
ignore_sql_errors=ignore_sql_errors,
|
|
483
|
-
only_response_times=only_response_times,
|
|
484
|
-
run_tests=run_tests,
|
|
485
|
-
as_standard=as_standard,
|
|
486
|
-
tests_to_run=tests_to_run,
|
|
487
|
-
tests_relative_change=tests_relative_change,
|
|
488
|
-
tests_to_sample_by_params=tests_to_sample_by_params,
|
|
489
|
-
tests_filter_by=tests_filter_by,
|
|
490
|
-
tests_failfast=tests_failfast,
|
|
491
|
-
tests_ignore_order=tests_ignore_order,
|
|
492
|
-
tests_validate_processed_bytes=tests_validate_processed_bytes,
|
|
493
|
-
override_datasource=override_datasource,
|
|
494
|
-
tests_check_requests_from_branch=tests_check_requests_from_branch,
|
|
495
|
-
fork_downstream=fork_downstream,
|
|
496
|
-
fork=fork,
|
|
497
|
-
)
|
|
498
|
-
await update_tags_in_resource(r, "pipe", tb_client)
|
|
499
|
-
elif r["resource"] == "datasources":
|
|
500
|
-
await new_ds(
|
|
501
|
-
r,
|
|
502
|
-
tb_client,
|
|
503
|
-
user_token,
|
|
504
|
-
force,
|
|
505
|
-
skip_confirmation=skip_confirmation,
|
|
506
|
-
current_ws=current_ws,
|
|
507
|
-
local_ws=local_ws,
|
|
508
|
-
fork_downstream=fork_downstream,
|
|
509
|
-
fork=fork,
|
|
510
|
-
build=build,
|
|
511
|
-
is_vendor=is_vendor,
|
|
512
|
-
)
|
|
513
|
-
await update_tags_in_resource(r, "datasource", tb_client)
|
|
514
|
-
else:
|
|
515
|
-
raise click.ClickException(FeedbackManager.error_unknown_resource(resource=r["resource"]))
|
|
516
|
-
|
|
517
|
-
|
|
518
309
|
def get_remote_resource_name_without_version(remote_resource_name: str) -> str:
|
|
519
310
|
"""
|
|
520
311
|
>>> get_remote_resource_name_without_version("r__datasource")
|
|
@@ -526,7 +526,7 @@ def format_parse_error(
|
|
|
526
526
|
message += f" found at position {adjusted_position - len(keyword)}"
|
|
527
527
|
else:
|
|
528
528
|
message += (
|
|
529
|
-
f" found {repr(table_structure[i]) if len(table_structure)>i else 'EOF'} at position {adjusted_position}"
|
|
529
|
+
f" found {repr(table_structure[i]) if len(table_structure) > i else 'EOF'} at position {adjusted_position}"
|
|
530
530
|
)
|
|
531
531
|
return message
|
|
532
532
|
|
|
@@ -148,7 +148,7 @@ async def format_engine(
|
|
|
148
148
|
else:
|
|
149
149
|
if node.get("engine", None):
|
|
150
150
|
empty = '""'
|
|
151
|
-
file_parts.append(f
|
|
151
|
+
file_parts.append(f"ENGINE {node['engine']['type']}" if node.get("engine", {}).get("type") else empty)
|
|
152
152
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
153
153
|
for arg in sorted(node["engine"].get("args", [])):
|
|
154
154
|
elem = ", ".join([x.strip() for x in arg[1].split(",")])
|
|
@@ -42,7 +42,7 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
|
|
|
42
42
|
if node_type == PipeNodeTypes.MATERIALIZED:
|
|
43
43
|
file_parts.append(node_type_upper)
|
|
44
44
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
45
|
-
file_parts.append(f
|
|
45
|
+
file_parts.append(f"DATASOURCE {node['datasource']}")
|
|
46
46
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
47
47
|
await format_engine(file_parts, node)
|
|
48
48
|
|
|
@@ -50,10 +50,10 @@ async def format_node_type(file_parts: List[str], node: Dict[str, Any]) -> List[
|
|
|
50
50
|
if node_type == PipeNodeTypes.COPY:
|
|
51
51
|
file_parts.append(node_type_upper)
|
|
52
52
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
53
|
-
file_parts.append(f
|
|
53
|
+
file_parts.append(f"TARGET_DATASOURCE {node['target_datasource']}")
|
|
54
54
|
if node.get("mode"):
|
|
55
55
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
56
|
-
file_parts.append(f
|
|
56
|
+
file_parts.append(f"COPY_MODE {node.get('mode')}")
|
|
57
57
|
|
|
58
58
|
if node.get(CopyParameters.COPY_SCHEDULE):
|
|
59
59
|
is_ondemand = node[CopyParameters.COPY_SCHEDULE].lower() == ON_DEMAND
|
|
@@ -112,7 +112,7 @@ async def format_node(
|
|
|
112
112
|
if item and not unroll_includes:
|
|
113
113
|
return
|
|
114
114
|
|
|
115
|
-
file_parts.append(f
|
|
115
|
+
file_parts.append(f"NODE {node['name'].strip()}")
|
|
116
116
|
file_parts.append(DATAFILE_NEW_LINE)
|
|
117
117
|
|
|
118
118
|
from collections import namedtuple
|