tinybird 0.0.1.dev67__py3-none-any.whl → 0.0.1.dev68__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/prompts.py +3 -3
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/create.py +42 -80
- tinybird/tb/modules/datafile/common.py +0 -2
- tinybird/tb/modules/deployment.py +47 -10
- tinybird/tb/modules/mock.py +11 -39
- tinybird/tb/modules/test.py +38 -70
- tinybird/tb/modules/watch.py +25 -12
- {tinybird-0.0.1.dev67.dist-info → tinybird-0.0.1.dev68.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev67.dist-info → tinybird-0.0.1.dev68.dist-info}/RECORD +13 -13
- {tinybird-0.0.1.dev67.dist-info → tinybird-0.0.1.dev68.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev67.dist-info → tinybird-0.0.1.dev68.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev67.dist-info → tinybird-0.0.1.dev68.dist-info}/top_level.txt +0 -0
tinybird/prompts.py
CHANGED
|
@@ -699,8 +699,8 @@ pipe_instructions = """
|
|
|
699
699
|
- Nodes can't have the same exact name as the Pipe they belong to.
|
|
700
700
|
- Avoid more than one node per pipe unless it is really necessary or requested by the user.
|
|
701
701
|
- No indentation is allowed for property names: DESCRIPTION, NODE, SQL, TYPE, etc.
|
|
702
|
-
- Allowed TYPE values are: endpoint, copy, materialized
|
|
703
|
-
- Add always the output node in the TYPE section.
|
|
702
|
+
- Allowed TYPE values are: endpoint, copy, materialized.
|
|
703
|
+
- Add always the output node in the TYPE section or in the last node of the pipe.
|
|
704
704
|
</pipe_file_instructions>
|
|
705
705
|
"""
|
|
706
706
|
|
|
@@ -788,7 +788,7 @@ When you need to work with resources or data in cloud, add always the --cloud fl
|
|
|
788
788
|
├── endpoints
|
|
789
789
|
├── fixtures
|
|
790
790
|
├── materializations
|
|
791
|
-
├──
|
|
791
|
+
├── pipes
|
|
792
792
|
└── tests
|
|
793
793
|
- The local development server will be available at http://localhost:80. Even if some response uses another base url, use always http://localhost:80.
|
|
794
794
|
- After every change in your .datasource, .pipe or .ndjson files, run `{base_command} build` to build the project locally.
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev68'
|
|
8
|
+
__revision__ = 'e28d545'
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -35,17 +35,9 @@ from tinybird.tb.modules.project import Project
|
|
|
35
35
|
)
|
|
36
36
|
@click.option("--rows", type=int, default=10, help="Number of events to send")
|
|
37
37
|
@click.option("--source", type=str, default="tb", help="Source of the command")
|
|
38
|
-
@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated resources")
|
|
39
38
|
@click.pass_context
|
|
40
39
|
@coro
|
|
41
|
-
async def create(
|
|
42
|
-
ctx: click.Context,
|
|
43
|
-
data: Optional[str],
|
|
44
|
-
prompt: Optional[str],
|
|
45
|
-
rows: int,
|
|
46
|
-
source: str,
|
|
47
|
-
skip: bool,
|
|
48
|
-
) -> None:
|
|
40
|
+
async def create(ctx: click.Context, data: Optional[str], prompt: Optional[str], rows: int, source: str) -> None:
|
|
49
41
|
"""Initialize a new project."""
|
|
50
42
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
51
43
|
local_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
@@ -80,7 +72,7 @@ async def create(
|
|
|
80
72
|
result = ""
|
|
81
73
|
if data or prompt:
|
|
82
74
|
click.echo(FeedbackManager.highlight(message="\n» Creating resources..."))
|
|
83
|
-
result = await create_resources(local_client, tb_client, user_token, data, prompt, folder
|
|
75
|
+
result = await create_resources(local_client, tb_client, user_token, data, prompt, folder)
|
|
84
76
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
85
77
|
|
|
86
78
|
if not already_has_cicd(folder):
|
|
@@ -125,7 +117,7 @@ async def create(
|
|
|
125
117
|
click.echo(FeedbackManager.error(message=f"Error: {str(e)}"))
|
|
126
118
|
|
|
127
119
|
|
|
128
|
-
PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "
|
|
120
|
+
PROJECT_PATHS = ("datasources", "endpoints", "materializations", "copies", "pipes", "fixtures", "tests")
|
|
129
121
|
|
|
130
122
|
|
|
131
123
|
def validate_project_structure(folder: str) -> bool:
|
|
@@ -166,7 +158,6 @@ async def create_resources(
|
|
|
166
158
|
data: Optional[str],
|
|
167
159
|
prompt: Optional[str],
|
|
168
160
|
folder: str,
|
|
169
|
-
skip: bool,
|
|
170
161
|
):
|
|
171
162
|
result = ""
|
|
172
163
|
folder_path = Path(folder)
|
|
@@ -215,73 +206,39 @@ TYPE ENDPOINT
|
|
|
215
206
|
]
|
|
216
207
|
)
|
|
217
208
|
llm = LLM(user_token=user_token, host=tb_client.host)
|
|
218
|
-
result =
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
resource_type = extract_xml(resource_xml, "type")
|
|
252
|
-
name = extract_xml(resource_xml, "name")
|
|
253
|
-
content = extract_xml(resource_xml, "content")
|
|
254
|
-
resource = {
|
|
255
|
-
"name": name,
|
|
256
|
-
"content": content,
|
|
257
|
-
}
|
|
258
|
-
if resource_type.lower() == "datasource":
|
|
259
|
-
datasources.append(resource)
|
|
260
|
-
elif resource_type.lower() == "pipe":
|
|
261
|
-
pipes.append(resource)
|
|
262
|
-
|
|
263
|
-
for ds in datasources:
|
|
264
|
-
content = ds["content"].replace("```", "")
|
|
265
|
-
filename = f"{ds['name']}.datasource"
|
|
266
|
-
datasource_path = generate_datafile(
|
|
267
|
-
content,
|
|
268
|
-
filename=filename,
|
|
269
|
-
data=None,
|
|
270
|
-
_format="ndjson",
|
|
271
|
-
force=True,
|
|
272
|
-
folder=folder,
|
|
273
|
-
)
|
|
274
|
-
generated_paths.append(datasource_path)
|
|
275
|
-
for pipe in pipes:
|
|
276
|
-
content = pipe["content"].replace("```", "")
|
|
277
|
-
pipe_path = generate_pipe_file(pipe["name"], content, folder)
|
|
278
|
-
generated_paths.append(pipe_path)
|
|
279
|
-
if skip:
|
|
280
|
-
break
|
|
281
|
-
iterations += 1
|
|
282
|
-
|
|
283
|
-
if iterations == 10:
|
|
284
|
-
click.echo(FeedbackManager.info(message="Too many iterations. Change the prompt and try again."))
|
|
209
|
+
result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt)
|
|
210
|
+
result = extract_xml(result, "response")
|
|
211
|
+
resources = parse_xml(result, "resource")
|
|
212
|
+
datasources = []
|
|
213
|
+
pipes = []
|
|
214
|
+
for resource_xml in resources:
|
|
215
|
+
resource_type = extract_xml(resource_xml, "type")
|
|
216
|
+
name = extract_xml(resource_xml, "name")
|
|
217
|
+
content = extract_xml(resource_xml, "content")
|
|
218
|
+
resource = {
|
|
219
|
+
"name": name,
|
|
220
|
+
"content": content,
|
|
221
|
+
}
|
|
222
|
+
if resource_type.lower() == "datasource":
|
|
223
|
+
datasources.append(resource)
|
|
224
|
+
elif resource_type.lower() == "pipe":
|
|
225
|
+
pipes.append(resource)
|
|
226
|
+
|
|
227
|
+
for ds in datasources:
|
|
228
|
+
content = ds["content"].replace("```", "")
|
|
229
|
+
filename = f"{ds['name']}.datasource"
|
|
230
|
+
generate_datafile(
|
|
231
|
+
content,
|
|
232
|
+
filename=filename,
|
|
233
|
+
data=None,
|
|
234
|
+
_format="ndjson",
|
|
235
|
+
force=True,
|
|
236
|
+
folder=folder,
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
for pipe in pipes:
|
|
240
|
+
content = pipe["content"].replace("```", "")
|
|
241
|
+
generate_pipe_file(pipe["name"], content, folder)
|
|
285
242
|
|
|
286
243
|
return result
|
|
287
244
|
|
|
@@ -313,14 +270,19 @@ def generate_pipe_file(name: str, content: str, folder: str) -> Path:
|
|
|
313
270
|
def is_sink(content: str) -> bool:
|
|
314
271
|
return re.search(r"TYPE sink", content, re.IGNORECASE) is not None
|
|
315
272
|
|
|
273
|
+
def is_endpoint(content: str) -> bool:
|
|
274
|
+
return re.search(r"TYPE endpoint", content, re.IGNORECASE) is not None
|
|
275
|
+
|
|
316
276
|
if is_copy(content):
|
|
317
277
|
pathname = "copies"
|
|
318
278
|
elif is_materialization(content):
|
|
319
279
|
pathname = "materializations"
|
|
320
280
|
elif is_sink(content):
|
|
321
281
|
pathname = "sinks"
|
|
322
|
-
|
|
282
|
+
elif is_endpoint(content):
|
|
323
283
|
pathname = "endpoints"
|
|
284
|
+
else:
|
|
285
|
+
pathname = "pipes"
|
|
324
286
|
|
|
325
287
|
base = Path(folder) / pathname
|
|
326
288
|
if not base.exists():
|
|
@@ -232,8 +232,6 @@ class Datafile:
|
|
|
232
232
|
node = self.nodes[0]
|
|
233
233
|
if "schema" not in node:
|
|
234
234
|
raise DatafileValidationError("SCHEMA is mandatory")
|
|
235
|
-
if "engine" not in node:
|
|
236
|
-
raise DatafileValidationError("ENGINE is mandatory")
|
|
237
235
|
else:
|
|
238
236
|
# We cannot validate a datafile whose kind is unknown
|
|
239
237
|
pass
|
|
@@ -58,6 +58,8 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
58
58
|
result = r.json()
|
|
59
59
|
logging.debug(json.dumps(result, indent=2))
|
|
60
60
|
|
|
61
|
+
click.echo(FeedbackManager.success(message="Deployment promotion successfully started"))
|
|
62
|
+
|
|
61
63
|
if wait:
|
|
62
64
|
while True:
|
|
63
65
|
TINYBIRD_API_URL = f"{host}/v1/deployments/{last_deployment.get('id')}"
|
|
@@ -67,12 +69,11 @@ def promote_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
67
69
|
|
|
68
70
|
last_deployment = result.get("deployment")
|
|
69
71
|
if last_deployment.get("status") == "deleted":
|
|
72
|
+
click.echo(FeedbackManager.success(message="Deployment promoted successfully"))
|
|
70
73
|
break
|
|
71
74
|
|
|
72
75
|
time.sleep(5)
|
|
73
76
|
|
|
74
|
-
click.echo(FeedbackManager.success(message="Deployment promoted successfully"))
|
|
75
|
-
|
|
76
77
|
|
|
77
78
|
# TODO(eclbg): This logic should be in the server, and there should be a dedicated endpoint for rolling back a
|
|
78
79
|
# deployment
|
|
@@ -117,6 +118,8 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
117
118
|
result = r.json()
|
|
118
119
|
logging.debug(json.dumps(result, indent=2))
|
|
119
120
|
|
|
121
|
+
click.echo(FeedbackManager.success(message="Deployment rollback successfully started"))
|
|
122
|
+
|
|
120
123
|
if wait:
|
|
121
124
|
while True:
|
|
122
125
|
TINYBIRD_API_URL = f"{host}/v1/deployments/{current_deployment.get('id')}"
|
|
@@ -126,11 +129,10 @@ def rollback_deployment(host: Optional[str], headers: dict, wait: bool) -> None:
|
|
|
126
129
|
|
|
127
130
|
current_deployment = result.get("deployment")
|
|
128
131
|
if current_deployment.get("status") == "deleted":
|
|
132
|
+
click.echo(FeedbackManager.success(message="Deployment rolled back successfully"))
|
|
129
133
|
break
|
|
130
134
|
time.sleep(5)
|
|
131
135
|
|
|
132
|
-
click.echo(FeedbackManager.success(message="Deployment rolled back successfully"))
|
|
133
|
-
|
|
134
136
|
|
|
135
137
|
@cli.group(name="deployment")
|
|
136
138
|
def deployment_group() -> None:
|
|
@@ -159,12 +161,18 @@ def deployment_group() -> None:
|
|
|
159
161
|
default=False,
|
|
160
162
|
help="Validate the deployment before creating it. Disabled by default.",
|
|
161
163
|
)
|
|
164
|
+
@click.option(
|
|
165
|
+
"--allow-remove-datasources/--no-allow-remove-datasources",
|
|
166
|
+
is_flag=True,
|
|
167
|
+
default=False,
|
|
168
|
+
help="Allow removing datasources. Disabled by default.",
|
|
169
|
+
)
|
|
162
170
|
@click.pass_context
|
|
163
|
-
def deployment_create(ctx: click.Context, wait: bool, auto: bool, check: bool) -> None:
|
|
171
|
+
def deployment_create(ctx: click.Context, wait: bool, auto: bool, check: bool, allow_remove_datasources: bool) -> None:
|
|
164
172
|
"""
|
|
165
173
|
Validate and deploy the project server side.
|
|
166
174
|
"""
|
|
167
|
-
create_deployment(ctx, wait, auto, check)
|
|
175
|
+
create_deployment(ctx, wait, auto, check, allow_remove_datasources)
|
|
168
176
|
|
|
169
177
|
|
|
170
178
|
@deployment_group.command(name="ls")
|
|
@@ -187,6 +195,9 @@ def deployment_ls(ctx: click.Context) -> None:
|
|
|
187
195
|
columns = ["ID", "Status", "Created at", "Live"]
|
|
188
196
|
table = []
|
|
189
197
|
for deployment in result.get("deployments"):
|
|
198
|
+
if deployment.get("id") == "0":
|
|
199
|
+
continue
|
|
200
|
+
|
|
190
201
|
table.append(
|
|
191
202
|
[
|
|
192
203
|
deployment.get("id"),
|
|
@@ -258,15 +269,27 @@ def deployment_rollback(ctx: click.Context, wait: bool) -> None:
|
|
|
258
269
|
default=False,
|
|
259
270
|
help="Validate the deployment before creating it. Disabled by default.",
|
|
260
271
|
)
|
|
272
|
+
@click.option(
|
|
273
|
+
"--allow-remove-datasources/--no-allow-remove-datasources",
|
|
274
|
+
is_flag=True,
|
|
275
|
+
default=False,
|
|
276
|
+
help="Allow removing datasources. Disabled by default.",
|
|
277
|
+
)
|
|
261
278
|
@click.pass_context
|
|
262
|
-
def deploy(ctx: click.Context, wait: bool, auto: bool, check: bool) -> None:
|
|
279
|
+
def deploy(ctx: click.Context, wait: bool, auto: bool, check: bool, allow_remove_datasources: bool) -> None:
|
|
263
280
|
"""
|
|
264
281
|
Deploy the project.
|
|
265
282
|
"""
|
|
266
|
-
create_deployment(ctx, wait, auto, check)
|
|
283
|
+
create_deployment(ctx, wait, auto, check, allow_remove_datasources)
|
|
267
284
|
|
|
268
285
|
|
|
269
|
-
def create_deployment(
|
|
286
|
+
def create_deployment(
|
|
287
|
+
ctx: click.Context,
|
|
288
|
+
wait: bool,
|
|
289
|
+
auto: bool,
|
|
290
|
+
check: Optional[bool] = None,
|
|
291
|
+
allow_remove_datasources: Optional[bool] = None,
|
|
292
|
+
) -> None:
|
|
270
293
|
# TODO: This code is duplicated in build_server.py
|
|
271
294
|
# Should be refactored to be shared
|
|
272
295
|
MULTIPART_BOUNDARY_DATA_PROJECT = "data_project://"
|
|
@@ -297,21 +320,31 @@ def create_deployment(ctx: click.Context, wait: bool, auto: bool, check: Optiona
|
|
|
297
320
|
if check:
|
|
298
321
|
click.echo(FeedbackManager.highlight(message="\n» Validating deployment...\n"))
|
|
299
322
|
params["check"] = "true"
|
|
323
|
+
if allow_remove_datasources:
|
|
324
|
+
params["allow_remove_datasources"] = "true"
|
|
300
325
|
r = requests.post(TINYBIRD_API_URL, files=files, headers=HEADERS, params=params)
|
|
301
326
|
result = r.json()
|
|
302
327
|
logging.debug(json.dumps(result, indent=2))
|
|
303
328
|
|
|
304
329
|
if check:
|
|
305
330
|
print_changes(result, project)
|
|
331
|
+
feedback = result.get("deployment", {}).get("feedback", [])
|
|
332
|
+
for f in feedback:
|
|
333
|
+
click.echo(
|
|
334
|
+
FeedbackManager.warning(message=f"△ {f.get('level')}: {f.get('resource')}: {f.get('message')}")
|
|
335
|
+
)
|
|
336
|
+
|
|
306
337
|
status = result.get("result")
|
|
307
338
|
if status == "success":
|
|
308
339
|
click.echo(FeedbackManager.success(message="\n✓ Deployment is valid"))
|
|
309
340
|
else:
|
|
310
341
|
click.echo(FeedbackManager.error(message="\n✗ Deployment is not valid"))
|
|
342
|
+
|
|
311
343
|
return
|
|
312
344
|
|
|
313
345
|
deploy_result = result.get("result")
|
|
314
346
|
if deploy_result == "success":
|
|
347
|
+
print_changes(result, project)
|
|
315
348
|
click.echo(FeedbackManager.success(message="Deployment submitted successfully"))
|
|
316
349
|
deployment = result.get("deployment")
|
|
317
350
|
feedback = deployment.get("feedback", [])
|
|
@@ -394,4 +427,8 @@ def print_changes(result: dict, project: Project) -> None:
|
|
|
394
427
|
for p in deployment.get("deleted_pipe_names", []):
|
|
395
428
|
resources.append(["deleted", p, project.get_resource_path(p, "pipe")])
|
|
396
429
|
|
|
397
|
-
|
|
430
|
+
if resources:
|
|
431
|
+
click.echo(FeedbackManager.highlight(message="\n» Changes to be deployed...\n"))
|
|
432
|
+
echo_safe_humanfriendly_tables_format_smart_table(resources, column_names=columns)
|
|
433
|
+
else:
|
|
434
|
+
click.echo(FeedbackManager.highlight(message="\n» No changes to be deployed\n"))
|
tinybird/tb/modules/mock.py
CHANGED
|
@@ -2,7 +2,6 @@ import glob
|
|
|
2
2
|
import logging
|
|
3
3
|
import os
|
|
4
4
|
from pathlib import Path
|
|
5
|
-
from typing import Optional
|
|
6
5
|
|
|
7
6
|
import click
|
|
8
7
|
|
|
@@ -27,10 +26,9 @@ from tinybird.tb.modules.project import Project
|
|
|
27
26
|
default="",
|
|
28
27
|
help="Extra context to use for data generation",
|
|
29
28
|
)
|
|
30
|
-
@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated data")
|
|
31
29
|
@click.pass_context
|
|
32
30
|
@coro
|
|
33
|
-
async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str
|
|
31
|
+
async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str) -> None:
|
|
34
32
|
"""Generate sample data for a datasource.
|
|
35
33
|
|
|
36
34
|
Args:
|
|
@@ -80,44 +78,18 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, skip
|
|
|
80
78
|
return
|
|
81
79
|
llm = LLM(user_token=user_token, host=user_client.host)
|
|
82
80
|
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
83
|
-
iterations = 0
|
|
84
|
-
history = ""
|
|
85
|
-
fixture_path: Optional[Path] = None
|
|
86
81
|
sql = ""
|
|
87
|
-
while iterations < 10:
|
|
88
|
-
feedback = ""
|
|
89
|
-
if iterations > 0:
|
|
90
|
-
feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
|
|
91
|
-
if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
|
|
92
|
-
break
|
|
93
|
-
else:
|
|
94
|
-
if iterations > 0:
|
|
95
|
-
if fixture_path:
|
|
96
|
-
fixture_path.unlink()
|
|
97
|
-
fixture_path = None
|
|
98
|
-
click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
|
|
99
82
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
history = (
|
|
111
|
-
history
|
|
112
|
-
+ f"""
|
|
113
|
-
<result_iteration_{iterations}>
|
|
114
|
-
{response}
|
|
115
|
-
</result_iteration_{iterations}>
|
|
116
|
-
"""
|
|
117
|
-
)
|
|
118
|
-
if skip:
|
|
119
|
-
break
|
|
120
|
-
iterations += 1
|
|
83
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
|
|
84
|
+
response = llm.ask(system_prompt=mock_prompt(rows), prompt=prompt)
|
|
85
|
+
sql = extract_xml(response, "sql")
|
|
86
|
+
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
87
|
+
data = result.get("data", [])[:rows]
|
|
88
|
+
persist_fixture(datasource_name, data, folder)
|
|
89
|
+
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.ndjson created"))
|
|
90
|
+
|
|
91
|
+
if os.environ.get("TB_DEBUG", "") != "":
|
|
92
|
+
logging.debug(sql)
|
|
121
93
|
|
|
122
94
|
click.echo(FeedbackManager.success(message=f"✓ Sample data for {datasource_name} created with {rows} rows"))
|
|
123
95
|
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -66,14 +66,9 @@ def test(ctx: click.Context) -> None:
|
|
|
66
66
|
@click.option(
|
|
67
67
|
"--prompt", type=str, default="Create a test for the selected pipe", help="Prompt to be used to create the test"
|
|
68
68
|
)
|
|
69
|
-
@click.option(
|
|
70
|
-
"--skip",
|
|
71
|
-
is_flag=True,
|
|
72
|
-
help="Skip the test creation process and only generate the test file",
|
|
73
|
-
)
|
|
74
69
|
@click.pass_context
|
|
75
70
|
@coro
|
|
76
|
-
async def test_create(ctx: click.Context, name_or_filename: str, prompt: str
|
|
71
|
+
async def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
77
72
|
"""
|
|
78
73
|
Create a test for an existing pipe
|
|
79
74
|
"""
|
|
@@ -114,74 +109,47 @@ async def test_create(ctx: click.Context, name_or_filename: str, prompt: str, sk
|
|
|
114
109
|
user_token = config.get_user_token()
|
|
115
110
|
if not user_token:
|
|
116
111
|
raise CLIException(FeedbackManager.error(message="No user token found"))
|
|
112
|
+
|
|
117
113
|
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
114
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Creating test for {pipe_name} endpoint..."))
|
|
115
|
+
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt)
|
|
116
|
+
response_xml = extract_xml(response_llm, "response")
|
|
117
|
+
tests_content = parse_xml(response_xml, "test")
|
|
118
118
|
|
|
119
|
-
|
|
120
|
-
history = ""
|
|
121
|
-
test_path: Optional[Path] = None
|
|
119
|
+
tests: List[Dict[str, Any]] = []
|
|
122
120
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
for test_content in tests_content:
|
|
142
|
-
test: Dict[str, Any] = {}
|
|
143
|
-
test["name"] = extract_xml(test_content, "name")
|
|
144
|
-
test["description"] = extract_xml(test_content, "description")
|
|
145
|
-
parameters_api = extract_xml(test_content, "parameters")
|
|
146
|
-
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
147
|
-
test["expected_result"] = ""
|
|
148
|
-
|
|
149
|
-
response = None
|
|
150
|
-
try:
|
|
151
|
-
response = await get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
152
|
-
except Exception:
|
|
153
|
-
pass
|
|
154
|
-
|
|
155
|
-
if response:
|
|
156
|
-
if response.status_code >= 400:
|
|
157
|
-
test["expected_http_status"] = response.status_code
|
|
158
|
-
test["expected_result"] = response.json()["error"]
|
|
159
|
-
else:
|
|
160
|
-
if "expected_http_status" in test:
|
|
161
|
-
del test["expected_http_status"]
|
|
162
|
-
test["expected_result"] = response.text or ""
|
|
163
|
-
|
|
164
|
-
tests.append(test)
|
|
165
|
-
|
|
166
|
-
if len(tests) > 0:
|
|
167
|
-
test_path = generate_test_file(pipe_name, tests, folder, mode="a")
|
|
168
|
-
for test in tests:
|
|
169
|
-
test_name = test["name"]
|
|
170
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
171
|
-
|
|
172
|
-
history = (
|
|
173
|
-
history
|
|
174
|
-
+ f"""
|
|
175
|
-
<result_iteration_{iterations}>
|
|
176
|
-
{response_xml}
|
|
177
|
-
</result_iteration_{iterations}>
|
|
178
|
-
"""
|
|
179
|
-
)
|
|
180
|
-
if skip:
|
|
181
|
-
break
|
|
182
|
-
iterations += 1
|
|
121
|
+
for test_content in tests_content:
|
|
122
|
+
test: Dict[str, Any] = {}
|
|
123
|
+
test["name"] = extract_xml(test_content, "name")
|
|
124
|
+
test["description"] = extract_xml(test_content, "description")
|
|
125
|
+
parameters_api = extract_xml(test_content, "parameters")
|
|
126
|
+
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
127
|
+
test["expected_result"] = ""
|
|
128
|
+
|
|
129
|
+
response = None
|
|
130
|
+
try:
|
|
131
|
+
response = await get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
132
|
+
except Exception:
|
|
133
|
+
pass
|
|
134
|
+
|
|
135
|
+
if response:
|
|
136
|
+
if response.status_code >= 400:
|
|
137
|
+
test["expected_http_status"] = response.status_code
|
|
138
|
+
test["expected_result"] = response.json()["error"]
|
|
183
139
|
else:
|
|
184
|
-
|
|
140
|
+
if "expected_http_status" in test:
|
|
141
|
+
del test["expected_http_status"]
|
|
142
|
+
test["expected_result"] = response.text or ""
|
|
143
|
+
|
|
144
|
+
tests.append(test)
|
|
145
|
+
|
|
146
|
+
if len(tests) > 0:
|
|
147
|
+
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
148
|
+
for test in tests:
|
|
149
|
+
test_name = test["name"]
|
|
150
|
+
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
151
|
+
else:
|
|
152
|
+
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
185
153
|
|
|
186
154
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
187
155
|
except Exception as e:
|
tinybird/tb/modules/watch.py
CHANGED
|
@@ -7,7 +7,7 @@ import click
|
|
|
7
7
|
from watchdog.events import (
|
|
8
8
|
DirDeletedEvent,
|
|
9
9
|
FileDeletedEvent,
|
|
10
|
-
|
|
10
|
+
FileSystemEventHandler,
|
|
11
11
|
)
|
|
12
12
|
from watchdog.observers import Observer
|
|
13
13
|
|
|
@@ -17,25 +17,23 @@ from tinybird.tb.modules.project import Project
|
|
|
17
17
|
from tinybird.tb.modules.shell import Shell
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
class WatchProjectHandler(
|
|
20
|
+
class WatchProjectHandler(FileSystemEventHandler):
|
|
21
21
|
def __init__(self, shell: Shell, project: Project, process: Callable):
|
|
22
22
|
self.shell = shell
|
|
23
23
|
self.project = project
|
|
24
24
|
self.process = process
|
|
25
25
|
self.datafiles = project.get_project_datafiles()
|
|
26
|
-
super().__init__(
|
|
27
|
-
patterns=[
|
|
28
|
-
f"{project.path}/**/*.datasource",
|
|
29
|
-
f"{project.path}/**/*.pipe",
|
|
30
|
-
f"{project.path}/fixtures/*.ndjson",
|
|
31
|
-
],
|
|
32
|
-
ignore_patterns=[f"{project.path}/vendor/"],
|
|
33
|
-
)
|
|
26
|
+
super().__init__()
|
|
34
27
|
|
|
35
28
|
def should_process(self, event: Any) -> Optional[str]:
|
|
36
29
|
if event.is_directory:
|
|
37
30
|
return None
|
|
38
31
|
|
|
32
|
+
valid_extensions = [".datasource", ".pipe", ".ndjson"]
|
|
33
|
+
|
|
34
|
+
if not any(event.src_path.endswith(ext) for ext in valid_extensions):
|
|
35
|
+
return None
|
|
36
|
+
|
|
39
37
|
if os.path.exists(event.src_path):
|
|
40
38
|
return event.src_path
|
|
41
39
|
|
|
@@ -91,13 +89,28 @@ class WatchProjectHandler(PatternMatchingEventHandler):
|
|
|
91
89
|
|
|
92
90
|
return None
|
|
93
91
|
|
|
94
|
-
def
|
|
92
|
+
def on_any_event(self, event):
|
|
93
|
+
if str(event.src_path).endswith("~"):
|
|
94
|
+
return None
|
|
95
|
+
|
|
96
|
+
if event.event_type == "modified":
|
|
97
|
+
self.modified(event)
|
|
98
|
+
elif event.event_type == "deleted":
|
|
99
|
+
self.deleted(event)
|
|
100
|
+
|
|
101
|
+
def created(self, event: Any) -> None:
|
|
102
|
+
if path := self.should_process(event):
|
|
103
|
+
filename = Path(path).name
|
|
104
|
+
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ New file detected: {filename}\n"))
|
|
105
|
+
self._process(path)
|
|
106
|
+
|
|
107
|
+
def modified(self, event: Any) -> None:
|
|
95
108
|
if path := self.should_process(event):
|
|
96
109
|
filename = Path(path).name
|
|
97
110
|
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
|
|
98
111
|
self._process(path)
|
|
99
112
|
|
|
100
|
-
def
|
|
113
|
+
def deleted(self, event: Union[DirDeletedEvent, FileDeletedEvent]) -> None:
|
|
101
114
|
filename = Path(str(event.src_path)).name
|
|
102
115
|
if event.is_directory:
|
|
103
116
|
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Deleted directory: {filename}\n"))
|
|
@@ -6,7 +6,7 @@ tinybird/context.py,sha256=A3GBApac9xO6hrAMJ1s9dMrI_ou9aKF84CdEjtPddMk,1417
|
|
|
6
6
|
tinybird/datatypes.py,sha256=XNypumfqNjsvLJ5iNXnbVHRvAJe0aQwI3lS6Cxox-e0,10979
|
|
7
7
|
tinybird/feedback_manager.py,sha256=ADuy1yC3fCFM_ii_Haphg8Gdzapa5pX9dlRTVbuUqxs,67990
|
|
8
8
|
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
9
|
-
tinybird/prompts.py,sha256=
|
|
9
|
+
tinybird/prompts.py,sha256=LQfE0ruY9A7z1iqtt6xTkW-1NlHbXl7LbKqzfdOsPxA,30612
|
|
10
10
|
tinybird/sql.py,sha256=LBi74GxhNAYTb6m2-KNGpAkguSKh7rcvBbERbE7nalA,46195
|
|
11
11
|
tinybird/sql_template.py,sha256=GmMLAI10MTqjQo9qztuQHLRWs67teozsWDxUBdvkAn4,93668
|
|
12
12
|
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
@@ -15,7 +15,7 @@ tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
|
15
15
|
tinybird/tornado_template.py,sha256=FL85SMPq2dH4JqKovmSbaolGdEzwOO91NqOzqXo2Qr0,41863
|
|
16
16
|
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
17
17
|
tinybird/ch_utils/engine.py,sha256=OXkBhlzGjZotjD0vaT-rFIbSGV4tpiHxE8qO_ip0SyQ,40454
|
|
18
|
-
tinybird/tb/__cli__.py,sha256=
|
|
18
|
+
tinybird/tb/__cli__.py,sha256=Ccq13PAaFIf4jhPqKWAcmtpsBFgPc30-vB_7SboquMU,251
|
|
19
19
|
tinybird/tb/cli.py,sha256=FD1pfbzu9YHJHEG6Vtn_EwPLTYhwqw-I6AxXeTaRHU8,926
|
|
20
20
|
tinybird/tb/modules/auth.py,sha256=EzRWFmwRkXNhUmRaruEVFLdkbUg8xMSix0cAWl5D4Jg,9029
|
|
21
21
|
tinybird/tb/modules/build.py,sha256=UN1d7EZ93VOlPCrtsay-KLgZnzxn2NCBDY3wvrUSP1Q,9198
|
|
@@ -24,9 +24,9 @@ tinybird/tb/modules/cli.py,sha256=D5E5nUMyDzj3tkvkKZdI9LHO9EGFuA1zt4GjI7NKLk0,15
|
|
|
24
24
|
tinybird/tb/modules/common.py,sha256=QjH9rugFYKkmGNd7bPBKXjrmROoRhHUtr4MA_X6n-wA,73238
|
|
25
25
|
tinybird/tb/modules/config.py,sha256=mie3oMVTf5YOUFEiLs88P16U4LkJafJjSpjwyAkFHog,10979
|
|
26
26
|
tinybird/tb/modules/copy.py,sha256=Aq6wh_wjRiyLQtEOKF9pKLPgJhSvbGTFWIw_LJB0t0U,5801
|
|
27
|
-
tinybird/tb/modules/create.py,sha256=
|
|
27
|
+
tinybird/tb/modules/create.py,sha256=I01JDENOyGKK0Umd2_1Om_nFGP8Uk9vxaOw7PygK02o,12302
|
|
28
28
|
tinybird/tb/modules/datasource.py,sha256=TQ4wSag3CCw34d54FEXPJFGLQNYyNqv2nQbU6QT9uAE,14725
|
|
29
|
-
tinybird/tb/modules/deployment.py,sha256=
|
|
29
|
+
tinybird/tb/modules/deployment.py,sha256=pVEdRWY31E4Blu9A9nVvnRj9zA4nZujBx77dRxP1sog,15706
|
|
30
30
|
tinybird/tb/modules/endpoint.py,sha256=7pOF4eLZ41vA5H6bKMCFZnOyKQKfdyytgwTO2Fcxtso,11907
|
|
31
31
|
tinybird/tb/modules/exceptions.py,sha256=4A2sSjCEqKUMqpP3WI00zouCWW4uLaghXXLZBSw04mY,3363
|
|
32
32
|
tinybird/tb/modules/feedback_manager.py,sha256=mrw5tdYycfvg6WLXlM0KIjfJardm_aNpnJkUg2vH0cA,68463
|
|
@@ -38,7 +38,7 @@ tinybird/tb/modules/local.py,sha256=_PIa-1M-72bv9rhLwqaNthJM1ZhvcjWXFChZAfEPXRs,
|
|
|
38
38
|
tinybird/tb/modules/local_common.py,sha256=W1fEnB1vBQ4YC5U1PdA0w0g3cTV78bQ5R-lRxdDj5-Y,2868
|
|
39
39
|
tinybird/tb/modules/login.py,sha256=EGxwVRmMX1Y7ZeCRyA8fqaCWpYYk7NvnZ3x_1g0NlYA,6063
|
|
40
40
|
tinybird/tb/modules/materialization.py,sha256=HQKRTH6lkcYiDQJihbFqF_in58ezXG4ggZ_7Ywp_nUM,5738
|
|
41
|
-
tinybird/tb/modules/mock.py,sha256=
|
|
41
|
+
tinybird/tb/modules/mock.py,sha256=FLFPQi68PhcKfjiaFXCjQUTqYsDTtsr4z5WQPn5XAa0,3929
|
|
42
42
|
tinybird/tb/modules/pipe.py,sha256=pH2KwgH6Xbvl3kT8vMelpKvT6bcyB4EKFDvGfOsxXbg,2418
|
|
43
43
|
tinybird/tb/modules/project.py,sha256=RkejMzY6OLWlJMckziZiZFJLnjFxSeQjagklXAYCzoQ,2945
|
|
44
44
|
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
@@ -46,16 +46,16 @@ tinybird/tb/modules/shell.py,sha256=6N5M08A0_xllfdZKAvKxrgGrG2CcS6EVtjXNDx6wuBE,
|
|
|
46
46
|
tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
|
|
47
47
|
tinybird/tb/modules/tag.py,sha256=anPmMUBc-TbFovlpFi8GPkKA18y7Y0GczMsMms5TZsU,3502
|
|
48
48
|
tinybird/tb/modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
49
|
-
tinybird/tb/modules/test.py,sha256=
|
|
49
|
+
tinybird/tb/modules/test.py,sha256=6R433tzlunNLlKdmkcfskAjBPj_v9ZvTaHFQNblay2E,11529
|
|
50
50
|
tinybird/tb/modules/token.py,sha256=A2I5wTUUmo0KfTI1BH6M6pqCQOd5dE4w2-Xaa1yM5PE,12810
|
|
51
|
-
tinybird/tb/modules/watch.py,sha256=
|
|
51
|
+
tinybird/tb/modules/watch.py,sha256=90FUrSD6cYfOi7GsEHfq_oTI6kap7uc8CbG0t3j_Tus,4953
|
|
52
52
|
tinybird/tb/modules/workspace.py,sha256=sfT9QkoeFlN7ndUXxyImp4a7EFEHjY9MlGlldOViz0Y,6404
|
|
53
53
|
tinybird/tb/modules/workspace_members.py,sha256=Ai6iCOzXX1zQ8q9iXIFSFHsBJlT-8Q28DaG5Ie-UweY,8726
|
|
54
54
|
tinybird/tb/modules/datafile/build.py,sha256=seGFSvmgyRrAM1-icsKBkuog3WccfGUYFTPT-xoA5W8,50940
|
|
55
55
|
tinybird/tb/modules/datafile/build_common.py,sha256=IXl-Z51zUi1dypV7meNenX0iu2UmowNeqgG6WHyMHlk,4562
|
|
56
56
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=4aP8_DYCRGghXntZSeWDNJxjps1QRVa7WHoYCzQwQts,17355
|
|
57
57
|
tinybird/tb/modules/datafile/build_pipe.py,sha256=Jgv3YKIvMfjPiSIdw1k2mpaoDdAWMiMRaSHwRgyI97E,28258
|
|
58
|
-
tinybird/tb/modules/datafile/common.py,sha256=
|
|
58
|
+
tinybird/tb/modules/datafile/common.py,sha256=LyVd_VtVoGEW_fal1G5yOpJu0cA9ZKraw1pPW-1ijI0,78364
|
|
59
59
|
tinybird/tb/modules/datafile/diff.py,sha256=-0J7PsBO64T7LOZSkZ4ZFHHCPvT7cKItnJkbz2PkndU,6754
|
|
60
60
|
tinybird/tb/modules/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
|
|
61
61
|
tinybird/tb/modules/datafile/fixture.py,sha256=XQpbppUQ4cow7dMzHs7OZBYBXA2PI5AOCmNdMDClbKo,833
|
|
@@ -74,8 +74,8 @@ tinybird/tb_cli_modules/config.py,sha256=6u6B5QCdiQLbJkCkwtnKGs9H3nP-KXXhC75mF7B
|
|
|
74
74
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
75
75
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
76
76
|
tinybird/tb_cli_modules/telemetry.py,sha256=iEGnMuCuNhvF6ln__j6X9MSTwL_0Hm-GgFHHHvhfknk,10466
|
|
77
|
-
tinybird-0.0.1.
|
|
78
|
-
tinybird-0.0.1.
|
|
79
|
-
tinybird-0.0.1.
|
|
80
|
-
tinybird-0.0.1.
|
|
81
|
-
tinybird-0.0.1.
|
|
77
|
+
tinybird-0.0.1.dev68.dist-info/METADATA,sha256=a9e8W_Cvpv2Itow-3_PLhvfYLzxj5d-jBTwbTAtfASY,2516
|
|
78
|
+
tinybird-0.0.1.dev68.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
79
|
+
tinybird-0.0.1.dev68.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
80
|
+
tinybird-0.0.1.dev68.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
81
|
+
tinybird-0.0.1.dev68.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|