tinybird 0.0.1.dev292__py3-none-any.whl → 0.0.1.dev294__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/ch_utils/constants.py +1 -0
- tinybird/datafile/parse_pipe.py +2 -2
- tinybird/sql_template.py +11 -3
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/agent/agent.py +14 -9
- tinybird/tb/modules/build.py +5 -3
- tinybird/tb/modules/build_common.py +140 -1
- tinybird/tb/modules/datasource.py +1 -1
- tinybird/tb/modules/feedback_manager.py +1 -1
- tinybird/tb/modules/local_common.py +14 -2
- tinybird/tb/modules/test.py +7 -4
- tinybird/tb/modules/test_common.py +19 -10
- {tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev294.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev294.dist-info}/RECORD +17 -17
- {tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev294.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev294.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev292.dist-info → tinybird-0.0.1.dev294.dist-info}/top_level.txt +0 -0
tinybird/ch_utils/constants.py
CHANGED
tinybird/datafile/parse_pipe.py
CHANGED
|
@@ -11,7 +11,7 @@ from tinybird.datafile.common import (
|
|
|
11
11
|
parse,
|
|
12
12
|
)
|
|
13
13
|
from tinybird.datafile.exceptions import IncludeFileNotFoundException, ParseException
|
|
14
|
-
from tinybird.sql_template import get_template_and_variables, render_sql_template
|
|
14
|
+
from tinybird.sql_template import get_template_and_variables, render_sql_template
|
|
15
15
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
16
16
|
from tinybird.tornado_template import UnClosedIfError
|
|
17
17
|
|
|
@@ -54,7 +54,7 @@ def parse_pipe(
|
|
|
54
54
|
if sql.strip()[0] == "%":
|
|
55
55
|
secrets_list: Optional[List[str]] = None
|
|
56
56
|
if secrets:
|
|
57
|
-
secrets_list =
|
|
57
|
+
secrets_list = list(secrets.keys())
|
|
58
58
|
# Setting test_mode=True to ignore errors on required parameters and
|
|
59
59
|
# secrets_in_test_mode=False to raise errors on missing secrets
|
|
60
60
|
sql, _, variable_warnings = render_sql_template(
|
tinybird/sql_template.py
CHANGED
|
@@ -1405,8 +1405,11 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1405
1405
|
namespace = {}
|
|
1406
1406
|
template_execution_results = TemplateExecutionResults()
|
|
1407
1407
|
for key in kwargs.get("tb_secrets", []):
|
|
1408
|
+
# Avoid double-prefixing if the key already has the tb_secret_ prefix
|
|
1408
1409
|
if is_secret_template_key(key):
|
|
1409
1410
|
template_execution_results.add_template_param(key)
|
|
1411
|
+
else:
|
|
1412
|
+
template_execution_results.add_template_param(secret_template_key(key))
|
|
1410
1413
|
|
|
1411
1414
|
if TB_SECRET_IN_TEST_MODE in kwargs:
|
|
1412
1415
|
template_execution_results[TB_SECRET_IN_TEST_MODE] = None
|
|
@@ -1415,15 +1418,20 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1415
1418
|
try:
|
|
1416
1419
|
key = secret_template_key(x)
|
|
1417
1420
|
if key in template_execution_results.template_params:
|
|
1421
|
+
# secret available: Always use workspace secret regardless of test mode
|
|
1418
1422
|
template_execution_results.add_ch_param(x)
|
|
1419
1423
|
return Symbol("{" + sqlescape(x) + ": String}")
|
|
1420
1424
|
else:
|
|
1425
|
+
# secret not available: Check test mode and defaults
|
|
1421
1426
|
is_test_mode = TB_SECRET_IN_TEST_MODE in template_execution_results
|
|
1422
|
-
if
|
|
1423
|
-
|
|
1424
|
-
elif default is not None:
|
|
1427
|
+
if default is not None:
|
|
1428
|
+
# Use provided default value
|
|
1425
1429
|
return default
|
|
1430
|
+
elif is_test_mode:
|
|
1431
|
+
# In test mode without default - return placeholder
|
|
1432
|
+
return Symbol("{" + sqlescape(x) + ": String}")
|
|
1426
1433
|
else:
|
|
1434
|
+
# Not in test mode, no secret, no default - raise error
|
|
1427
1435
|
raise SQLTemplateException(
|
|
1428
1436
|
f"Cannot access secret '{x}'. Check the secret exists in the Workspace and the token has the required scope."
|
|
1429
1437
|
)
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev294'
|
|
8
|
+
__revision__ = 'b3b73ff'
|
|
@@ -373,7 +373,7 @@ class TinybirdAgent:
|
|
|
373
373
|
execute_query_local=partial(execute_query_local, config=config),
|
|
374
374
|
request_endpoint_cloud=partial(request_endpoint_cloud, config=config),
|
|
375
375
|
request_endpoint_local=partial(request_endpoint_local, config=config),
|
|
376
|
-
build_project_test=partial(build_project_test, project=project, client=test_client),
|
|
376
|
+
build_project_test=partial(build_project_test, project=project, client=test_client, config=config),
|
|
377
377
|
get_pipe_data_test=partial(get_pipe_data_test, client=test_client),
|
|
378
378
|
get_datasource_datafile_cloud=partial(get_datasource_datafile_cloud, config=config),
|
|
379
379
|
get_datasource_datafile_local=partial(get_datasource_datafile_local, config=config),
|
|
@@ -382,7 +382,7 @@ class TinybirdAgent:
|
|
|
382
382
|
get_connection_datafile_cloud=partial(get_connection_datafile_cloud, config=config),
|
|
383
383
|
get_connection_datafile_local=partial(get_connection_datafile_local, config=config),
|
|
384
384
|
get_project_files=project.get_project_files,
|
|
385
|
-
run_tests=partial(run_tests, project=project, client=test_client),
|
|
385
|
+
run_tests=partial(run_tests, project=project, client=test_client, config=config),
|
|
386
386
|
folder=folder,
|
|
387
387
|
thinking_animation=self.thinking_animation,
|
|
388
388
|
workspace_id=self.workspace_id,
|
|
@@ -482,9 +482,9 @@ def run_agent(
|
|
|
482
482
|
if yes:
|
|
483
483
|
update_cli()
|
|
484
484
|
click.echo(FeedbackManager.highlight(message="» Initializing Tinybird Code..."))
|
|
485
|
-
token = config.get("token",
|
|
486
|
-
host = config.get("host",
|
|
487
|
-
user_token = config.get("user_token",
|
|
485
|
+
token = config.get("token", "")
|
|
486
|
+
host = config.get("host", "")
|
|
487
|
+
user_token = config.get("user_token", "")
|
|
488
488
|
workspace_id = config.get("id", "")
|
|
489
489
|
workspace_name = config.get("name", "")
|
|
490
490
|
try:
|
|
@@ -501,8 +501,8 @@ def run_agent(
|
|
|
501
501
|
login(host, auth_host="https://cloud.tinybird.co", workspace=None, interactive=False, method="browser")
|
|
502
502
|
cli_config = CLIConfig.get_project_config()
|
|
503
503
|
config = {**config, **cli_config.to_dict()}
|
|
504
|
-
token = cli_config.get_token()
|
|
505
|
-
user_token = cli_config.get_user_token()
|
|
504
|
+
token = cli_config.get_token() or ""
|
|
505
|
+
user_token = cli_config.get_user_token() or ""
|
|
506
506
|
host = cli_config.get_host()
|
|
507
507
|
workspace_id = cli_config.get("id", "")
|
|
508
508
|
workspace_name = cli_config.get("name", "")
|
|
@@ -652,6 +652,7 @@ def build_project(
|
|
|
652
652
|
build_error = build_process(
|
|
653
653
|
project=project,
|
|
654
654
|
tb_client=client,
|
|
655
|
+
config=config,
|
|
655
656
|
watch=False,
|
|
656
657
|
silent=silent,
|
|
657
658
|
exit_on_error=False,
|
|
@@ -664,11 +665,13 @@ def build_project(
|
|
|
664
665
|
def build_project_test(
|
|
665
666
|
client: TinyB,
|
|
666
667
|
project: Project,
|
|
668
|
+
config: dict[str, Any],
|
|
667
669
|
silent: bool = False,
|
|
668
670
|
) -> None:
|
|
669
671
|
build_error = build_process(
|
|
670
672
|
project=project,
|
|
671
673
|
tb_client=client,
|
|
674
|
+
config=config,
|
|
672
675
|
watch=False,
|
|
673
676
|
silent=silent,
|
|
674
677
|
exit_on_error=False,
|
|
@@ -821,9 +824,11 @@ def get_connection_datafile_local(config: dict[str, Any], connection_name: str)
|
|
|
821
824
|
return "Connection not found"
|
|
822
825
|
|
|
823
826
|
|
|
824
|
-
def run_tests(
|
|
827
|
+
def run_tests(
|
|
828
|
+
client: TinyB, project: Project, config: dict[str, Any], pipe_name: Optional[str] = None
|
|
829
|
+
) -> Optional[str]:
|
|
825
830
|
try:
|
|
826
|
-
return run_tests_common(name=(pipe_name,) if pipe_name else (), project=project, client=client)
|
|
831
|
+
return run_tests_common(name=(pipe_name,) if pipe_name else (), project=project, client=client, config=config)
|
|
827
832
|
except SystemExit as e:
|
|
828
833
|
raise Exception(e.args[0])
|
|
829
834
|
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -34,6 +34,7 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
34
34
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
35
35
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
36
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
|
+
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
37
38
|
if obj["env"] == "cloud":
|
|
38
39
|
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
39
40
|
|
|
@@ -47,12 +48,12 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
47
48
|
)
|
|
48
49
|
|
|
49
50
|
click.echo(FeedbackManager.highlight_building_project())
|
|
50
|
-
process(project=project, tb_client=tb_client, watch=False)
|
|
51
|
+
process(project=project, tb_client=tb_client, watch=False, config=config)
|
|
51
52
|
if watch:
|
|
52
53
|
run_watch(
|
|
53
54
|
project=project,
|
|
54
55
|
tb_client=tb_client,
|
|
55
|
-
process=partial(process, project=project, tb_client=tb_client, watch=True),
|
|
56
|
+
process=partial(process, project=project, tb_client=tb_client, watch=True, config=config),
|
|
56
57
|
)
|
|
57
58
|
|
|
58
59
|
|
|
@@ -65,6 +66,7 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
|
|
|
65
66
|
return dev_cloud(ctx)
|
|
66
67
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
67
68
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
69
|
+
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
68
70
|
build_status = BuildStatus()
|
|
69
71
|
if ui:
|
|
70
72
|
server_thread = threading.Thread(
|
|
@@ -75,7 +77,7 @@ def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
|
|
|
75
77
|
time.sleep(0.5)
|
|
76
78
|
|
|
77
79
|
click.echo(FeedbackManager.highlight_building_project())
|
|
78
|
-
process(project=project, tb_client=tb_client, watch=True, build_status=build_status)
|
|
80
|
+
process(project=project, tb_client=tb_client, watch=True, config=config, build_status=build_status)
|
|
79
81
|
run_watch(
|
|
80
82
|
project=project,
|
|
81
83
|
tb_client=tb_client,
|
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
3
|
import time
|
|
4
|
+
from copy import deepcopy
|
|
4
5
|
from pathlib import Path
|
|
5
|
-
from typing import Optional
|
|
6
|
+
from typing import Any, Optional
|
|
6
7
|
from urllib.parse import urlencode, urljoin
|
|
7
8
|
|
|
8
9
|
import click
|
|
9
10
|
import requests
|
|
10
11
|
|
|
12
|
+
from tinybird.datafile.parse_datasource import parse_datasource
|
|
11
13
|
from tinybird.tb.client import TinyB
|
|
12
14
|
from tinybird.tb.modules.common import push_data, sys_exit
|
|
13
15
|
from tinybird.tb.modules.datafile.fixture import FixtureExtension, get_fixture_dir, persist_fixture
|
|
14
16
|
from tinybird.tb.modules.dev_server import BuildStatus
|
|
15
17
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
18
|
+
from tinybird.tb.modules.local_common import get_local_tokens
|
|
16
19
|
from tinybird.tb.modules.project import Project
|
|
17
20
|
from tinybird.tb.modules.shell import print_table_formatted
|
|
18
21
|
|
|
@@ -21,6 +24,7 @@ def process(
|
|
|
21
24
|
project: Project,
|
|
22
25
|
tb_client: TinyB,
|
|
23
26
|
watch: bool,
|
|
27
|
+
config: dict[str, Any],
|
|
24
28
|
file_changed: Optional[str] = None,
|
|
25
29
|
diff: Optional[str] = None,
|
|
26
30
|
silent: bool = False,
|
|
@@ -29,6 +33,12 @@ def process(
|
|
|
29
33
|
load_fixtures: bool = True,
|
|
30
34
|
) -> Optional[str]:
|
|
31
35
|
time_start = time.time()
|
|
36
|
+
|
|
37
|
+
# Build vendored workspaces before build
|
|
38
|
+
build_vendored_workspaces(project=project, tb_client=tb_client, config=config)
|
|
39
|
+
# Ensure SHARED_WITH workspaces exist before build
|
|
40
|
+
build_shared_with_workspaces(project=project, tb_client=tb_client, config=config)
|
|
41
|
+
|
|
32
42
|
build_failed = False
|
|
33
43
|
build_error: Optional[str] = None
|
|
34
44
|
build_result: Optional[bool] = None
|
|
@@ -320,3 +330,132 @@ def echo_changes(project: Project, changes: list[str], extension: str, status: s
|
|
|
320
330
|
if path_str:
|
|
321
331
|
path_str = path_str.replace(f"{project.folder}/", "")
|
|
322
332
|
click.echo(FeedbackManager.info(message=f"✓ {path_str} {status}"))
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def find_workspace_or_create(user_client: TinyB, workspace_name: str) -> Optional[str]:
|
|
336
|
+
# Get a client scoped to the vendored workspace using the user token
|
|
337
|
+
ws_token = None
|
|
338
|
+
org_id = None
|
|
339
|
+
try:
|
|
340
|
+
# Fetch org id and workspaces with tokens
|
|
341
|
+
info = user_client.user_workspaces_with_organization(version="v1")
|
|
342
|
+
org_id = info.get("organization_id")
|
|
343
|
+
workspaces = info.get("workspaces", [])
|
|
344
|
+
found = next((w for w in workspaces if w.get("name") == workspace_name), None)
|
|
345
|
+
if found:
|
|
346
|
+
ws_token = found.get("token")
|
|
347
|
+
# If still not found, try the generic listing
|
|
348
|
+
if not ws_token:
|
|
349
|
+
workspaces_full = user_client.user_workspaces_and_branches(version="v1")
|
|
350
|
+
created_ws = next(
|
|
351
|
+
(w for w in workspaces_full.get("workspaces", []) if w.get("name") == workspace_name), None
|
|
352
|
+
)
|
|
353
|
+
if created_ws:
|
|
354
|
+
ws_token = created_ws.get("token")
|
|
355
|
+
except Exception:
|
|
356
|
+
ws_token = None
|
|
357
|
+
|
|
358
|
+
# If workspace doesn't exist, try to create it and fetch its token
|
|
359
|
+
if not ws_token:
|
|
360
|
+
try:
|
|
361
|
+
user_client.create_workspace(workspace_name, assign_to_organization_id=org_id, version="v1")
|
|
362
|
+
# Fetch token for newly created workspace
|
|
363
|
+
info_after = user_client.user_workspaces_and_branches(version="v1")
|
|
364
|
+
created = next((w for w in info_after.get("workspaces", []) if w.get("name") == workspace_name), None)
|
|
365
|
+
ws_token = created.get("token") if created else None
|
|
366
|
+
except Exception as e:
|
|
367
|
+
click.echo(
|
|
368
|
+
FeedbackManager.warning(
|
|
369
|
+
message=(f"Skipping vendored workspace '{workspace_name}': unable to create or resolve token ({e})")
|
|
370
|
+
)
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
return ws_token
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def build_vendored_workspaces(project: Project, tb_client: TinyB, config: dict[str, Any]) -> None:
|
|
377
|
+
"""Build each vendored workspace under project.vendor_path if present.
|
|
378
|
+
|
|
379
|
+
Directory structure expected: vendor/<workspace_name>/<data_project_inside>
|
|
380
|
+
Each top-level directory under vendor is treated as a separate workspace
|
|
381
|
+
whose project files will be built using that workspace's token.
|
|
382
|
+
"""
|
|
383
|
+
try:
|
|
384
|
+
vendor_root = Path(project.vendor_path)
|
|
385
|
+
|
|
386
|
+
if not vendor_root.exists() or not vendor_root.is_dir():
|
|
387
|
+
return
|
|
388
|
+
|
|
389
|
+
tokens = get_local_tokens()
|
|
390
|
+
user_token = tokens["user_token"]
|
|
391
|
+
user_client = deepcopy(tb_client)
|
|
392
|
+
user_client.token = user_token
|
|
393
|
+
|
|
394
|
+
# Iterate over vendored workspace folders
|
|
395
|
+
for ws_dir in sorted([p for p in vendor_root.iterdir() if p.is_dir()]):
|
|
396
|
+
workspace_name = ws_dir.name
|
|
397
|
+
ws_token = find_workspace_or_create(user_client, workspace_name)
|
|
398
|
+
|
|
399
|
+
if not ws_token:
|
|
400
|
+
click.echo(
|
|
401
|
+
FeedbackManager.warning(
|
|
402
|
+
message=f"Skipping vendored workspace '{workspace_name}': could not resolve token after creation"
|
|
403
|
+
)
|
|
404
|
+
)
|
|
405
|
+
continue
|
|
406
|
+
|
|
407
|
+
# Build using a client scoped to the vendor workspace token
|
|
408
|
+
vendor_client = deepcopy(tb_client)
|
|
409
|
+
vendor_client.token = ws_token
|
|
410
|
+
vendor_project = Project(folder=str(ws_dir), workspace_name=workspace_name, max_depth=project.max_depth)
|
|
411
|
+
|
|
412
|
+
# Do not exit on error to allow main project to continue
|
|
413
|
+
process(
|
|
414
|
+
project=vendor_project,
|
|
415
|
+
tb_client=vendor_client,
|
|
416
|
+
watch=False,
|
|
417
|
+
silent=True,
|
|
418
|
+
exit_on_error=True,
|
|
419
|
+
load_fixtures=True,
|
|
420
|
+
config=config,
|
|
421
|
+
)
|
|
422
|
+
except Exception as e:
|
|
423
|
+
# Never break the main build due to vendored build errors
|
|
424
|
+
click.echo(FeedbackManager.error_exception(error=e))
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def build_shared_with_workspaces(project: Project, tb_client: TinyB, config: dict[str, Any]) -> None:
|
|
428
|
+
"""Scan project for .datasource files and ensure SHARED_WITH workspaces exist."""
|
|
429
|
+
|
|
430
|
+
try:
|
|
431
|
+
# Gather SHARED_WITH workspace names from all .datasource files
|
|
432
|
+
datasource_files = project.get_datasource_files()
|
|
433
|
+
shared_ws_names = set()
|
|
434
|
+
|
|
435
|
+
for filename in datasource_files:
|
|
436
|
+
try:
|
|
437
|
+
doc = parse_datasource(filename).datafile
|
|
438
|
+
for ws_name in doc.shared_with or []:
|
|
439
|
+
shared_ws_names.add(ws_name)
|
|
440
|
+
except Exception:
|
|
441
|
+
# Ignore parse errors here; they'll be handled during the main process()
|
|
442
|
+
continue
|
|
443
|
+
|
|
444
|
+
if not shared_ws_names:
|
|
445
|
+
return
|
|
446
|
+
|
|
447
|
+
# Need a user token to list/create workspaces
|
|
448
|
+
tokens = get_local_tokens()
|
|
449
|
+
user_token = tokens.get("user_token")
|
|
450
|
+
if not user_token:
|
|
451
|
+
click.echo(FeedbackManager.info_skipping_shared_with_entry())
|
|
452
|
+
return
|
|
453
|
+
|
|
454
|
+
user_client = deepcopy(tb_client)
|
|
455
|
+
user_client.token = user_token
|
|
456
|
+
|
|
457
|
+
# Ensure each SHARED_WITH workspace exists
|
|
458
|
+
for ws_name in sorted(shared_ws_names):
|
|
459
|
+
find_workspace_or_create(user_client, ws_name)
|
|
460
|
+
except Exception as e:
|
|
461
|
+
click.echo(FeedbackManager.error_exception(error=e))
|
|
@@ -931,7 +931,7 @@ def datasource_create(
|
|
|
931
931
|
default=True,
|
|
932
932
|
):
|
|
933
933
|
click.echo(FeedbackManager.gray(message="» Building project..."))
|
|
934
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
934
|
+
build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
|
|
935
935
|
click.echo(FeedbackManager.success(message="✓ Build completed!"))
|
|
936
936
|
connections = client.connections("kafka")
|
|
937
937
|
|
|
@@ -1209,7 +1209,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
|
|
|
1209
1209
|
|
|
1210
1210
|
debug_running_file = print_message("** Running {file}", bcolors.CGREY)
|
|
1211
1211
|
|
|
1212
|
-
highlight_building_project = info_highlight_message("
|
|
1212
|
+
highlight_building_project = info_highlight_message("» Building project...")
|
|
1213
1213
|
|
|
1214
1214
|
success = success_message("{message}")
|
|
1215
1215
|
info = info_message("{message}")
|
|
@@ -5,6 +5,7 @@ import os
|
|
|
5
5
|
import re
|
|
6
6
|
import subprocess
|
|
7
7
|
import time
|
|
8
|
+
import uuid
|
|
8
9
|
from typing import Any, Dict, Optional
|
|
9
10
|
|
|
10
11
|
import boto3
|
|
@@ -56,6 +57,17 @@ def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, si
|
|
|
56
57
|
if path:
|
|
57
58
|
user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
|
|
58
59
|
if test:
|
|
60
|
+
# delete any Tinybird_Local_Test_* workspace
|
|
61
|
+
user_workspaces = requests.get(
|
|
62
|
+
f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?with_organization=true&token={admin_token}"
|
|
63
|
+
).json()
|
|
64
|
+
local_workspaces = user_workspaces.get("workspaces", [])
|
|
65
|
+
for ws in local_workspaces:
|
|
66
|
+
if ws["name"].startswith("Tinybird_Local_Test_"):
|
|
67
|
+
requests.delete(
|
|
68
|
+
f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
|
|
69
|
+
)
|
|
70
|
+
|
|
59
71
|
ws_name = get_test_workspace_name(path)
|
|
60
72
|
else:
|
|
61
73
|
ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
|
|
@@ -104,8 +116,8 @@ def get_build_workspace_name(path: str) -> str:
|
|
|
104
116
|
|
|
105
117
|
|
|
106
118
|
def get_test_workspace_name(path: str) -> str:
|
|
107
|
-
|
|
108
|
-
return f"Tinybird_Local_Test_{
|
|
119
|
+
random_folder_suffix = str(uuid.uuid4()).replace("-", "_")
|
|
120
|
+
return f"Tinybird_Local_Test_{random_folder_suffix}"
|
|
109
121
|
|
|
110
122
|
|
|
111
123
|
def get_local_tokens() -> Dict[str, str]:
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
from typing import Tuple
|
|
6
|
+
from typing import Any, Tuple
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
|
|
@@ -34,7 +34,8 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
34
34
|
"""
|
|
35
35
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
36
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
|
-
|
|
37
|
+
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
38
|
+
create_test(name_or_filename, prompt, project, client, config=config)
|
|
38
39
|
|
|
39
40
|
|
|
40
41
|
@test.command(
|
|
@@ -46,7 +47,8 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
46
47
|
def test_update(ctx: click.Context, pipe: str) -> None:
|
|
47
48
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
48
49
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
49
|
-
|
|
50
|
+
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
51
|
+
update_test(pipe, project, client, config=config)
|
|
50
52
|
|
|
51
53
|
|
|
52
54
|
@test.command(
|
|
@@ -58,4 +60,5 @@ def test_update(ctx: click.Context, pipe: str) -> None:
|
|
|
58
60
|
def run_tests_command(ctx: click.Context, name: Tuple[str, ...]) -> None:
|
|
59
61
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
60
62
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
61
|
-
|
|
63
|
+
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
64
|
+
run_tests(name, project, client, config=config)
|
|
@@ -18,7 +18,6 @@ from tinybird.prompts import test_create_prompt
|
|
|
18
18
|
from tinybird.tb.client import TinyB
|
|
19
19
|
from tinybird.tb.modules.build_common import process as build_project
|
|
20
20
|
from tinybird.tb.modules.common import sys_exit
|
|
21
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
22
21
|
from tinybird.tb.modules.exceptions import CLITestException
|
|
23
22
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
24
23
|
from tinybird.tb.modules.llm import LLM
|
|
@@ -55,7 +54,7 @@ def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Opti
|
|
|
55
54
|
|
|
56
55
|
|
|
57
56
|
def create_test(
|
|
58
|
-
name_or_filename: str, prompt: str, project: Project, client: TinyB, preview: bool = False
|
|
57
|
+
name_or_filename: str, prompt: str, project: Project, client: TinyB, config: dict[str, Any], preview: bool = False
|
|
59
58
|
) -> list[dict[str, Any]]:
|
|
60
59
|
"""
|
|
61
60
|
Create a test for an existing pipe
|
|
@@ -64,11 +63,12 @@ def create_test(
|
|
|
64
63
|
|
|
65
64
|
try:
|
|
66
65
|
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
67
|
-
build_error = build_project(
|
|
66
|
+
build_error = build_project(
|
|
67
|
+
project=project, tb_client=client, watch=False, silent=True, exit_on_error=False, config=config
|
|
68
|
+
)
|
|
68
69
|
if build_error:
|
|
69
70
|
raise Exception(build_error)
|
|
70
71
|
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
71
|
-
config = CLIConfig.get_project_config()
|
|
72
72
|
folder = project.folder
|
|
73
73
|
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
74
74
|
pipe_name = pipe_path.stem
|
|
@@ -82,11 +82,11 @@ def create_test(
|
|
|
82
82
|
content=pipe_content,
|
|
83
83
|
parameters=parameters or "No parameters",
|
|
84
84
|
)
|
|
85
|
-
user_token = config.
|
|
85
|
+
user_token = config.get("user_token")
|
|
86
86
|
if not user_token:
|
|
87
87
|
raise Exception("No user token found")
|
|
88
88
|
|
|
89
|
-
llm = LLM(user_token=user_token, host=config.
|
|
89
|
+
llm = LLM(user_token=user_token, host=config.get("host") or "")
|
|
90
90
|
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
91
91
|
response_xml = extract_xml(response_llm, "response")
|
|
92
92
|
tests_content = parse_xml(response_xml, "test")
|
|
@@ -142,11 +142,13 @@ def dump_tests(tests: List[Dict[str, Any]]) -> str:
|
|
|
142
142
|
return yaml_str.replace("- name:", "\n- name:")
|
|
143
143
|
|
|
144
144
|
|
|
145
|
-
def update_test(pipe: str, project: Project, client: TinyB) -> None:
|
|
145
|
+
def update_test(pipe: str, project: Project, client: TinyB, config: dict[str, Any]) -> None:
|
|
146
146
|
try:
|
|
147
147
|
folder = project.folder
|
|
148
148
|
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
149
|
-
build_error = build_project(
|
|
149
|
+
build_error = build_project(
|
|
150
|
+
project=project, tb_client=client, watch=False, silent=True, exit_on_error=False, config=config
|
|
151
|
+
)
|
|
150
152
|
if build_error:
|
|
151
153
|
raise Exception(build_error)
|
|
152
154
|
|
|
@@ -190,11 +192,18 @@ def update_test(pipe: str, project: Project, client: TinyB) -> None:
|
|
|
190
192
|
cleanup_test_workspace(client, project.folder)
|
|
191
193
|
|
|
192
194
|
|
|
193
|
-
def run_tests(name: Tuple[str, ...], project: Project, client: TinyB) -> Optional[str]:
|
|
195
|
+
def run_tests(name: Tuple[str, ...], project: Project, client: TinyB, config: dict[str, Any]) -> Optional[str]:
|
|
194
196
|
full_error = ""
|
|
195
197
|
try:
|
|
196
198
|
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
197
|
-
build_error = build_project(
|
|
199
|
+
build_error = build_project(
|
|
200
|
+
project=project,
|
|
201
|
+
tb_client=client,
|
|
202
|
+
watch=False,
|
|
203
|
+
silent=True,
|
|
204
|
+
exit_on_error=False,
|
|
205
|
+
config=config,
|
|
206
|
+
)
|
|
198
207
|
if build_error:
|
|
199
208
|
raise Exception(build_error)
|
|
200
209
|
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
@@ -6,25 +6,25 @@ tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
|
6
6
|
tinybird/prompts.py,sha256=HoDv9TxPiP8v2XoGTWYxP133dK9CEbXVv4XE5IT339c,45483
|
|
7
7
|
tinybird/service_datasources.py,sha256=o6Az3T6OvcChR_7GXRu7sJH173KzGg1Gv-dNd0bI_vY,46085
|
|
8
8
|
tinybird/sql.py,sha256=7pkwQMUVy0CH5zFgLMZyCx1BeaJSQYC05EmOb9vO3Ls,48694
|
|
9
|
-
tinybird/sql_template.py,sha256=
|
|
9
|
+
tinybird/sql_template.py,sha256=rNePArEmx5GemxW6GJAiNnln0NRWncGgFe0EbC8G2f4,102463
|
|
10
10
|
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
11
11
|
tinybird/sql_toolset.py,sha256=Y2_fQrbk5GSNoRncAmRS_snpV61XQbiOy4uYkeF6pr4,19767
|
|
12
12
|
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
13
13
|
tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
|
|
14
|
-
tinybird/ch_utils/constants.py,sha256=
|
|
14
|
+
tinybird/ch_utils/constants.py,sha256=eQhAxN1vBBzFCP0-UGbBjkuCp4jrFzZRUkfVcKQ2osg,4231
|
|
15
15
|
tinybird/ch_utils/engine.py,sha256=4X1B-iuhdW_mxKnX_m3iCsxgP9RPVgR75g7yH1vsJ6A,40851
|
|
16
16
|
tinybird/datafile/common.py,sha256=hr9Qv4Xot21l2dW6xKPa5_JUu8pnGuDNnVpDf8pgthc,105659
|
|
17
17
|
tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1wnI,556
|
|
18
18
|
tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
|
|
19
19
|
tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
|
|
20
|
-
tinybird/datafile/parse_pipe.py,sha256=
|
|
21
|
-
tinybird/tb/__cli__.py,sha256
|
|
20
|
+
tinybird/datafile/parse_pipe.py,sha256=8e9LMecSQVWHC4AXf8cdxoQ1nxUR4fTObYxTctO_EXQ,3816
|
|
21
|
+
tinybird/tb/__cli__.py,sha256=WOliD3wiTxHrfZ79e4-jU5sf1wFngafUTW5PChhWgY4,247
|
|
22
22
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
23
23
|
tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
|
|
24
24
|
tinybird/tb/client.py,sha256=IQRaInDjOwr9Fzaz3_xXc3aUGqh94tM2lew7IZbB9eM,53733
|
|
25
25
|
tinybird/tb/config.py,sha256=mhMTGnMB5KcxGoh3dewIr2Jjsa6pHE183gCPAQWyp6o,3973
|
|
26
|
-
tinybird/tb/modules/build.py,sha256
|
|
27
|
-
tinybird/tb/modules/build_common.py,sha256=
|
|
26
|
+
tinybird/tb/modules/build.py,sha256=-aKn1fkoMoCnh0a6iQr-9EQuApVDeRYUrwcH3BosU-w,7828
|
|
27
|
+
tinybird/tb/modules/build_common.py,sha256=O9WXbZHRjBUwczCr2CGlXIkLQOiwAEJGL9TiY-0eEh8,18627
|
|
28
28
|
tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
|
|
29
29
|
tinybird/tb/modules/cli.py,sha256=-hkwg2ZIWPtldkLkgdY7RYWacOJtkMx58eqqYzMoMfw,17793
|
|
30
30
|
tinybird/tb/modules/common.py,sha256=zY0PFDQVWjhu2MivBtUmcXtAlv7imgO-RG_PUK37XvI,83659
|
|
@@ -32,21 +32,21 @@ tinybird/tb/modules/config.py,sha256=gK7rgaWTDd4ZKCrNEg_Uemr26EQjqWt6TjyQKujxOws
|
|
|
32
32
|
tinybird/tb/modules/connection.py,sha256=axp8Fny1_4PSLJGN4UF6WygyRbQtM3Lbt6thxHKTxzw,17790
|
|
33
33
|
tinybird/tb/modules/copy.py,sha256=dPZkcIDvxjJrlQUIvToO0vsEEEs4EYumbNV77-BzNoU,4404
|
|
34
34
|
tinybird/tb/modules/create.py,sha256=pJxHXG69c9Z_21s-7VuJ3RZOF_nJU51LEwiAkvI3dZY,23251
|
|
35
|
-
tinybird/tb/modules/datasource.py,sha256=
|
|
35
|
+
tinybird/tb/modules/datasource.py,sha256=7Ugj62Nyj6eBB7v9Jyny44KenYSce7rVs2h09N_0hyE,41812
|
|
36
36
|
tinybird/tb/modules/deployment.py,sha256=v0layOmG0IMnuXc3RT39mpGfa5M8yPlrL9F089fJFCo,15964
|
|
37
37
|
tinybird/tb/modules/deployment_common.py,sha256=ZX27oUw7u1ld20qiXH69j7m0DImOc5nckABZ37BPAn4,20406
|
|
38
38
|
tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
|
|
39
39
|
tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
|
|
40
40
|
tinybird/tb/modules/endpoint.py,sha256=ksRj6mfDb9Xv63PhTkV_uKSosgysHElqagg3RTt21Do,11958
|
|
41
41
|
tinybird/tb/modules/exceptions.py,sha256=KEj3E-zN7Z28igvptlngXIjlHR9vAJk59Vs4zpwm4vA,5469
|
|
42
|
-
tinybird/tb/modules/feedback_manager.py,sha256=
|
|
42
|
+
tinybird/tb/modules/feedback_manager.py,sha256=duigLI2UIeMzP5IOa7XUty23NRvNssm2ZAKicuun3Pg,78130
|
|
43
43
|
tinybird/tb/modules/info.py,sha256=F5vY4kHS_kyO2uSBKac92HoOb447oDeRlzpwtAHTuKc,6872
|
|
44
44
|
tinybird/tb/modules/infra.py,sha256=JE9oLIyF4bi_JBoe-BgZ5HhKp_lQgSihuSV1KIS02Qs,32709
|
|
45
45
|
tinybird/tb/modules/job.py,sha256=wBsnu8UPTOha2rkLvucgmw4xYv73ubmui3eeSIF68ZM,3107
|
|
46
46
|
tinybird/tb/modules/llm.py,sha256=fPBBCmM3KlCksLlgJkg4joDn6y3H5QjDzE-Pm4YNf7E,1782
|
|
47
47
|
tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
|
|
48
48
|
tinybird/tb/modules/local.py,sha256=kW3IHwJPvhBsa1eMh_xzow9Az3yYpHthkzsLSHeP5HE,6512
|
|
49
|
-
tinybird/tb/modules/local_common.py,sha256=
|
|
49
|
+
tinybird/tb/modules/local_common.py,sha256=Otvx7R8FjqEpI_5FS8BaraINChpF9XNuw5LW_WTL2XA,18474
|
|
50
50
|
tinybird/tb/modules/login.py,sha256=zerXZqIv15pbFk5XRt746xGcVnp01YmL_403byBf4jQ,1245
|
|
51
51
|
tinybird/tb/modules/login_common.py,sha256=CKXD_BjivhGUmBtNbLTJwzQDr6885C72XPZjo0GLLvI,12010
|
|
52
52
|
tinybird/tb/modules/logout.py,sha256=sniI4JNxpTrVeRCp0oGJuQ3yRerG4hH5uz6oBmjv724,1009
|
|
@@ -63,14 +63,14 @@ tinybird/tb/modules/shell.py,sha256=_9PaKkkh6ZjkixVtKNAtoCPqXMXMn1aQJM_Xzirn7ZM,
|
|
|
63
63
|
tinybird/tb/modules/sink.py,sha256=dK2s__my0ePIUYrqBzhPSgdWN9rbpvP1G4dT7DJzz80,3865
|
|
64
64
|
tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
|
|
65
65
|
tinybird/tb/modules/telemetry.py,sha256=T9gtsQffWqG_4hRBaUJPzOfMkPwz7mH-R6Bn1XRYViA,11482
|
|
66
|
-
tinybird/tb/modules/test.py,sha256=
|
|
67
|
-
tinybird/tb/modules/test_common.py,sha256=
|
|
66
|
+
tinybird/tb/modules/test.py,sha256=gi7fH6zavFp1jDs3SoRWdf9uBiKjp1hdOPAEleg7H-Y,2145
|
|
67
|
+
tinybird/tb/modules/test_common.py,sha256=pvHVXZb9xGosh4h_updz13qsIg3pyAtroFnjjrxaoPU,14673
|
|
68
68
|
tinybird/tb/modules/token.py,sha256=ZhW_o7XCr90wJRhMN6816vyo_TVfnzPXyIhrhzQ7oZ0,13807
|
|
69
69
|
tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,8819
|
|
70
70
|
tinybird/tb/modules/workspace.py,sha256=tCP1zZMwBhLRGm22TGfpSd4cHvQLAS1o_azIXv_r6uw,11172
|
|
71
71
|
tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
|
|
72
72
|
tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
|
|
73
|
-
tinybird/tb/modules/agent/agent.py,sha256=
|
|
73
|
+
tinybird/tb/modules/agent/agent.py,sha256=TqmXTsEGgt3rx6hPlRegCLdVxUzYF5P-x4OuOvIqFg0,37863
|
|
74
74
|
tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
|
|
75
75
|
tinybird/tb/modules/agent/banner.py,sha256=l6cO5Fi7lbVKp-GsBP8jf3IkjOWxg2jpAt9NBCy0WR8,4085
|
|
76
76
|
tinybird/tb/modules/agent/command_agent.py,sha256=0Z08rQsir59zQAr-kkOvsKIFpIBsBSTGJJ1VgqqF5WA,3654
|
|
@@ -121,8 +121,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
121
121
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
122
122
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
123
123
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
124
|
-
tinybird-0.0.1.
|
|
125
|
-
tinybird-0.0.1.
|
|
126
|
-
tinybird-0.0.1.
|
|
127
|
-
tinybird-0.0.1.
|
|
128
|
-
tinybird-0.0.1.
|
|
124
|
+
tinybird-0.0.1.dev294.dist-info/METADATA,sha256=xt6XB_652KuU-2KoOStFfaIviXpS1Luml6EySb92MHU,1845
|
|
125
|
+
tinybird-0.0.1.dev294.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
126
|
+
tinybird-0.0.1.dev294.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
127
|
+
tinybird-0.0.1.dev294.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
128
|
+
tinybird-0.0.1.dev294.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|