tinybird 0.0.1.dev258__py3-none-any.whl → 0.0.1.dev259__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/prompts.py +1 -0
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/agent/agent.py +14 -0
- tinybird/tb/modules/agent/prompts.py +35 -3
- tinybird/tb/modules/agent/tools/test.py +120 -0
- tinybird/tb/modules/agent/utils.py +1 -0
- tinybird/tb/modules/project.py +24 -1
- tinybird/tb/modules/test.py +12 -273
- tinybird/tb/modules/test_common.py +295 -0
- {tinybird-0.0.1.dev258.dist-info → tinybird-0.0.1.dev259.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev258.dist-info → tinybird-0.0.1.dev259.dist-info}/RECORD +14 -12
- {tinybird-0.0.1.dev258.dist-info → tinybird-0.0.1.dev259.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev258.dist-info → tinybird-0.0.1.dev259.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev258.dist-info → tinybird-0.0.1.dev259.dist-info}/top_level.txt +0 -0
tinybird/prompts.py
CHANGED
|
@@ -376,6 +376,7 @@ You are a Tinybird expert. You will be given a pipe containing different nodes w
|
|
|
376
376
|
- If there are no parameters, you can omit parameters and generate a single test.
|
|
377
377
|
- The format of the parameters is the following: ?param1=value1¶m2=value2¶m3=value3
|
|
378
378
|
- If some parameters are provided by the user and you need to use them, preserve in the same format as they were provided, like case sensitive.
|
|
379
|
+
- If user provides the current test content, use it to generate the new test content.
|
|
379
380
|
</instructions>
|
|
380
381
|
|
|
381
382
|
This is an example of a test with parameters:
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev259'
|
|
8
|
+
__revision__ = '5b5805f'
|
|
@@ -32,6 +32,8 @@ from tinybird.tb.modules.agent.tools.mock import mock
|
|
|
32
32
|
from tinybird.tb.modules.agent.tools.plan import plan
|
|
33
33
|
from tinybird.tb.modules.agent.tools.preview_datafile import preview_datafile
|
|
34
34
|
from tinybird.tb.modules.agent.tools.request_endpoint import request_endpoint
|
|
35
|
+
from tinybird.tb.modules.agent.tools.test import create_tests as create_tests_tool
|
|
36
|
+
from tinybird.tb.modules.agent.tools.test import run_tests as run_tests_tool
|
|
35
37
|
from tinybird.tb.modules.agent.utils import AgentRunCancelled, TinybirdAgentContext, show_input
|
|
36
38
|
from tinybird.tb.modules.build_common import process as build_process
|
|
37
39
|
from tinybird.tb.modules.common import _analyze, _get_tb_client, echo_safe_humanfriendly_tables_format_pretty_table
|
|
@@ -43,6 +45,7 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
|
43
45
|
from tinybird.tb.modules.login_common import login
|
|
44
46
|
from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
|
|
45
47
|
from tinybird.tb.modules.project import Project
|
|
48
|
+
from tinybird.tb.modules.test_common import run_tests as run_tests_common
|
|
46
49
|
|
|
47
50
|
|
|
48
51
|
class TinybirdAgent:
|
|
@@ -93,6 +96,8 @@ class TinybirdAgent:
|
|
|
93
96
|
Tool(execute_query, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
94
97
|
Tool(request_endpoint, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
95
98
|
Tool(diff_resource, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
99
|
+
Tool(create_tests_tool, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
100
|
+
Tool(run_tests_tool, docstring_format="google", require_parameter_descriptions=True, takes_ctx=True),
|
|
96
101
|
],
|
|
97
102
|
# history_processors=[self._keep_recent_messages],
|
|
98
103
|
)
|
|
@@ -152,6 +157,7 @@ class TinybirdAgent:
|
|
|
152
157
|
get_connection_datafile_cloud=partial(get_connection_datafile_cloud, config=config),
|
|
153
158
|
get_connection_datafile_local=partial(get_connection_datafile_local, config=config),
|
|
154
159
|
get_project_files=project.get_project_files,
|
|
160
|
+
run_tests=partial(run_tests, project=project, config=config),
|
|
155
161
|
folder=folder,
|
|
156
162
|
thinking_animation=self.thinking_animation,
|
|
157
163
|
workspace_name=self.project.workspace_name,
|
|
@@ -495,3 +501,11 @@ def get_connection_datafile_local(config: dict[str, Any], connection_name: str)
|
|
|
495
501
|
return local_client.connection_file(connection_name)
|
|
496
502
|
except Exception:
|
|
497
503
|
return "Connection not found"
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def run_tests(config: dict[str, Any], project: Project, pipe_name: Optional[str] = None) -> None:
|
|
507
|
+
local_client = get_tinybird_local_client(config, test=True, silent=True)
|
|
508
|
+
try:
|
|
509
|
+
run_tests_common(name=(pipe_name,) if pipe_name else (), project=project, client=local_client)
|
|
510
|
+
except SystemExit as e:
|
|
511
|
+
raise Exception(e.args[0])
|
|
@@ -16,6 +16,7 @@ from tinybird.prompts import (
|
|
|
16
16
|
pipe_instructions,
|
|
17
17
|
s3_connection_example,
|
|
18
18
|
sink_pipe_instructions,
|
|
19
|
+
test_instructions,
|
|
19
20
|
)
|
|
20
21
|
from tinybird.tb.modules.project import Project
|
|
21
22
|
|
|
@@ -114,6 +115,7 @@ datafile_instructions = """
|
|
|
114
115
|
def resources_prompt(project: Project) -> str:
|
|
115
116
|
files = project.get_project_files()
|
|
116
117
|
fixture_files = project.get_fixture_files()
|
|
118
|
+
test_files = project.get_test_files()
|
|
117
119
|
|
|
118
120
|
resources_content = "# Existing resources in the project:\n"
|
|
119
121
|
if files:
|
|
@@ -146,7 +148,22 @@ def resources_prompt(project: Project) -> str:
|
|
|
146
148
|
else:
|
|
147
149
|
fixture_content += "No fixture files found"
|
|
148
150
|
|
|
149
|
-
|
|
151
|
+
test_content = "# Test files in the project:\n"
|
|
152
|
+
if test_files:
|
|
153
|
+
tests: list[dict[str, Any]] = []
|
|
154
|
+
for filename in test_files:
|
|
155
|
+
file_path = Path(filename)
|
|
156
|
+
test = {
|
|
157
|
+
"path": str(file_path.relative_to(project.folder)),
|
|
158
|
+
"name": file_path.stem,
|
|
159
|
+
"content": file_path.read_text(),
|
|
160
|
+
}
|
|
161
|
+
tests.append(test)
|
|
162
|
+
test_content = format_as_xml(tests, root_tag="tests", item_tag="test")
|
|
163
|
+
else:
|
|
164
|
+
test_content += "No test files found"
|
|
165
|
+
|
|
166
|
+
return resources_content + "\n" + fixture_content + "\n" + test_content
|
|
150
167
|
|
|
151
168
|
|
|
152
169
|
def get_resource_type(path: Path) -> str:
|
|
@@ -550,6 +567,7 @@ You have access to the following tools:
|
|
|
550
567
|
13. `execute_query` - Execute a query against Tinybird Cloud or Local.
|
|
551
568
|
13. `request_endpoint` - Request an endpoint against Tinybird Cloud or Local.
|
|
552
569
|
14. `diff_resource` - Diff the content of a resource in Tinybird Cloud vs Tinybird Local vs Project local file.
|
|
570
|
+
15. `create_tests` - Create tests for an endpoint.
|
|
553
571
|
|
|
554
572
|
# When creating or updating datafiles:
|
|
555
573
|
1. Use `plan` tool to plan the creation or update of resources.
|
|
@@ -612,22 +630,26 @@ Kafka: {kafka_connection_example}
|
|
|
612
630
|
S3: {s3_connection_example}
|
|
613
631
|
GCS: {gcs_connection_example}
|
|
614
632
|
|
|
615
|
-
# When executing a query or
|
|
633
|
+
# When executing a query or calling an endpoint:
|
|
616
634
|
- You need to be sure that the selected resource is updated to the last version in the environment you are working on.
|
|
617
635
|
- Use `diff_resource` tool to compare the content of the resource to compare the differences between environments.
|
|
618
636
|
- Project local file is the source of truth.
|
|
619
637
|
- If the resource is not present or updated to the last version in Tinybird Local, it means you need to build the project.
|
|
620
638
|
- If the resource is not present or updated to the last version in Tinybird Cloud, it means you need to deploy the project.
|
|
639
|
+
- If exploring an endpoint, the response is empty. You can query the tables to understand what data is available.
|
|
640
|
+
|
|
621
641
|
|
|
622
642
|
# How to use apppend tools:
|
|
623
643
|
- Use append as part of the creation of a new landing datasource if the user provided a file or an external url
|
|
624
644
|
- Use append if user explicitly asks for it
|
|
625
|
-
- Do not append data if user requests to test an endpoint
|
|
645
|
+
- Do not append data if user requests to test an endpoint or call an endpoint.
|
|
646
|
+
- Do not append data as consequence of an empty response from the endpoint or a query.
|
|
626
647
|
|
|
627
648
|
# How to use `mock` tool:
|
|
628
649
|
- Use `mock` tool as part of the creation of a new landing datasource if the user did not provided a file or an external url
|
|
629
650
|
- Use `mock` tool if user explicitly asks for it
|
|
630
651
|
- Do not use `mock` tool if user requests to test an endpoint.
|
|
652
|
+
- Do not use `mock` tool as consequence of an empty response from the endpoint or a query.
|
|
631
653
|
|
|
632
654
|
# When sharing endpoints paths or urls:
|
|
633
655
|
- Use `get_openapi_definition` tool to get the url of the endpoint and parameters available.
|
|
@@ -639,6 +661,16 @@ GCS: {gcs_connection_example}
|
|
|
639
661
|
- `DateTime` parameters accept values in format `YYYY-MM-DD HH:MM:SS`
|
|
640
662
|
- `Date` parameters accept values in format `YYYYMMDD`
|
|
641
663
|
|
|
664
|
+
# Working with tests:
|
|
665
|
+
{test_instructions}
|
|
666
|
+
<dev_notes>
|
|
667
|
+
- Create tests only when the user explicitly asks for it with prompts like "Create tests for this endpoint" or "Create tests for this pipe".
|
|
668
|
+
- If the user asks for "testing an endpoint" or "call an endpoint", just request to the endpoint.
|
|
669
|
+
- The data that the tests are using is the data provided in the fixtures folder.
|
|
670
|
+
- Querying data or requesting endpoints won't return the data that the tests are using.
|
|
671
|
+
- MANDATORY: Before creating the test, analyze the fixture files that the tables of the endpoint are using so you can create relevant tests.
|
|
672
|
+
</dev_notes>
|
|
673
|
+
|
|
642
674
|
# Info
|
|
643
675
|
Today is {datetime.now().strftime("%Y-%m-%d")}
|
|
644
676
|
"""
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
from pydantic_ai import RunContext
|
|
6
|
+
|
|
7
|
+
from tinybird.tb.modules.agent.utils import (
|
|
8
|
+
AgentRunCancelled,
|
|
9
|
+
TinybirdAgentContext,
|
|
10
|
+
create_terminal_box,
|
|
11
|
+
show_confirmation,
|
|
12
|
+
show_input,
|
|
13
|
+
)
|
|
14
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def create_tests(ctx: RunContext[TinybirdAgentContext], pipe_name: str, test_content: str) -> str:
|
|
18
|
+
"""Given a pipe name, create or update a test for it
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
pipe_name (str): The pipe name to create a test for. Required.
|
|
22
|
+
test_content (str): The content of the test. Required.
|
|
23
|
+
|
|
24
|
+
Returns:
|
|
25
|
+
str: If the test was created or updated and the result of running the tests.
|
|
26
|
+
"""
|
|
27
|
+
running_tests = False
|
|
28
|
+
try:
|
|
29
|
+
ctx.deps.thinking_animation.stop()
|
|
30
|
+
path = Path(ctx.deps.folder) / "tests" / f"{pipe_name}.yaml"
|
|
31
|
+
current_test_content: Optional[str] = None
|
|
32
|
+
if path.exists():
|
|
33
|
+
current_test_content = path.read_text()
|
|
34
|
+
|
|
35
|
+
if current_test_content:
|
|
36
|
+
content = create_terminal_box(current_test_content, new_content=test_content, title=path.name)
|
|
37
|
+
else:
|
|
38
|
+
content = create_terminal_box(test_content, title=path.name)
|
|
39
|
+
|
|
40
|
+
click.echo(content)
|
|
41
|
+
action_text = "Create" if not current_test_content else "Update"
|
|
42
|
+
confirmation = show_confirmation(
|
|
43
|
+
title=f"{action_text} '{path.name}'?",
|
|
44
|
+
skip_confirmation=ctx.deps.dangerously_skip_permissions,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
if confirmation == "review":
|
|
48
|
+
feedback = show_input(ctx.deps.workspace_name)
|
|
49
|
+
ctx.deps.thinking_animation.start()
|
|
50
|
+
return f"User did not confirm the proposed changes and gave the following feedback: {feedback}"
|
|
51
|
+
|
|
52
|
+
folder_path = path.parent
|
|
53
|
+
folder_path.mkdir(parents=True, exist_ok=True)
|
|
54
|
+
path.touch(exist_ok=True)
|
|
55
|
+
path.write_text(test_content)
|
|
56
|
+
action_text = "created" if not current_test_content else "updated"
|
|
57
|
+
click.echo(FeedbackManager.success(message=f"✓ {path.name} {action_text}"))
|
|
58
|
+
running_tests = True
|
|
59
|
+
test_output = ctx.deps.run_tests(pipe_name=pipe_name)
|
|
60
|
+
click.echo(test_output)
|
|
61
|
+
ctx.deps.thinking_animation.start()
|
|
62
|
+
return f"Test {action_text} for '{pipe_name}' endpoint in {path} and ran successfully\n{test_output}"
|
|
63
|
+
except AgentRunCancelled as e:
|
|
64
|
+
raise e
|
|
65
|
+
except Exception as e:
|
|
66
|
+
error_message = str(e).replace("test_error__error__", "")
|
|
67
|
+
ctx.deps.thinking_animation.stop()
|
|
68
|
+
if not running_tests:
|
|
69
|
+
click.echo(FeedbackManager.error(message=error_message))
|
|
70
|
+
ctx.deps.thinking_animation.start()
|
|
71
|
+
if running_tests:
|
|
72
|
+
return f"Test {action_text} for '{pipe_name}' endpoint in {path} but there were errors running the tests: {error_message}"
|
|
73
|
+
return f"Error creating test for '{pipe_name}' endpoint: {error_message}"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def run_tests(ctx: RunContext[TinybirdAgentContext], pipe_name: Optional[str] = None) -> str:
|
|
77
|
+
"""Run tests for a given pipe name or all tests in the project
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
pipe_name (Optional[str]): The pipe name to run tests for. If not provided, all tests in the project will be run.
|
|
81
|
+
|
|
82
|
+
Returns:
|
|
83
|
+
str: The result of running the tests.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
ctx.deps.thinking_animation.stop()
|
|
88
|
+
path = Path(ctx.deps.folder) / "tests" / f"{pipe_name}.yaml"
|
|
89
|
+
|
|
90
|
+
title = f"Run tests for '{pipe_name}'?" if pipe_name else "Run all tests in the project?"
|
|
91
|
+
confirmation = show_confirmation(
|
|
92
|
+
title=title,
|
|
93
|
+
skip_confirmation=ctx.deps.dangerously_skip_permissions,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
if confirmation == "review":
|
|
97
|
+
feedback = show_input(ctx.deps.workspace_name)
|
|
98
|
+
ctx.deps.thinking_animation.start()
|
|
99
|
+
return f"User did not confirm the proposed changes and gave the following feedback: {feedback}"
|
|
100
|
+
|
|
101
|
+
test_output = ctx.deps.run_tests(pipe_name=pipe_name)
|
|
102
|
+
click.echo(test_output)
|
|
103
|
+
ctx.deps.thinking_animation.start()
|
|
104
|
+
if pipe_name:
|
|
105
|
+
return f"Tests for '{pipe_name}' endpoint in {path} and ran successfully\n{test_output}"
|
|
106
|
+
else:
|
|
107
|
+
return f"All tests in the project ran successfully\n{test_output}"
|
|
108
|
+
except AgentRunCancelled as e:
|
|
109
|
+
raise e
|
|
110
|
+
except Exception as e:
|
|
111
|
+
error_message = str(e)
|
|
112
|
+
test_exit_code = "test_error__error__"
|
|
113
|
+
test_error = test_exit_code in error_message
|
|
114
|
+
ctx.deps.thinking_animation.stop()
|
|
115
|
+
if not test_error:
|
|
116
|
+
click.echo(FeedbackManager.error(message=error_message))
|
|
117
|
+
else:
|
|
118
|
+
error_message = error_message.replace(test_exit_code, "")
|
|
119
|
+
ctx.deps.thinking_animation.start()
|
|
120
|
+
return f"Error running tests: {error_message}"
|
|
@@ -52,6 +52,7 @@ class TinybirdAgentContext(BaseModel):
|
|
|
52
52
|
get_pipe_datafile_local: Callable[..., str]
|
|
53
53
|
get_connection_datafile_cloud: Callable[..., str]
|
|
54
54
|
get_connection_datafile_local: Callable[..., str]
|
|
55
|
+
run_tests: Callable[..., Optional[str]]
|
|
55
56
|
dangerously_skip_permissions: bool
|
|
56
57
|
token: str
|
|
57
58
|
user_token: str
|
tinybird/tb/modules/project.py
CHANGED
|
@@ -24,6 +24,10 @@ class Project:
|
|
|
24
24
|
def vendor_path(self) -> str:
|
|
25
25
|
return f"{self.path}/vendor"
|
|
26
26
|
|
|
27
|
+
@property
|
|
28
|
+
def tests_path(self) -> str:
|
|
29
|
+
return f"{self.path}/tests"
|
|
30
|
+
|
|
27
31
|
def get_files(self, extension: str) -> List[str]:
|
|
28
32
|
project_files: List[str] = []
|
|
29
33
|
for level in range(self.max_depth):
|
|
@@ -52,13 +56,32 @@ class Project:
|
|
|
52
56
|
|
|
53
57
|
def get_fixture_files(self) -> List[str]:
|
|
54
58
|
fixture_files: List[str] = []
|
|
55
|
-
for extension in [
|
|
59
|
+
for extension in [
|
|
60
|
+
"csv",
|
|
61
|
+
"csv.gz",
|
|
62
|
+
"ndjson",
|
|
63
|
+
"ndjson.gz",
|
|
64
|
+
"jsonl",
|
|
65
|
+
"jsonl.gz",
|
|
66
|
+
"json",
|
|
67
|
+
"json.gz",
|
|
68
|
+
"parquet",
|
|
69
|
+
"parquet.gz",
|
|
70
|
+
]:
|
|
56
71
|
for fixture_file in self.get_files(extension):
|
|
57
72
|
if self.vendor_path in fixture_file:
|
|
58
73
|
continue
|
|
59
74
|
fixture_files.append(fixture_file)
|
|
60
75
|
return fixture_files
|
|
61
76
|
|
|
77
|
+
def get_test_files(self) -> List[str]:
|
|
78
|
+
test_files: List[str] = []
|
|
79
|
+
for test_file in self.get_files("yaml"):
|
|
80
|
+
if self.vendor_path in test_file or self.tests_path not in test_file:
|
|
81
|
+
continue
|
|
82
|
+
test_files.append(test_file)
|
|
83
|
+
return test_files
|
|
84
|
+
|
|
62
85
|
def get_resource_path(self, resource_name: str, resource_type: str) -> str:
|
|
63
86
|
full_path = next(
|
|
64
87
|
(p for p in self.get_project_files() if p.endswith("/" + resource_name + f".{resource_type}")), ""
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -3,57 +3,14 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import
|
|
7
|
-
import glob
|
|
8
|
-
import sys
|
|
9
|
-
import urllib.parse
|
|
10
|
-
from copy import deepcopy
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
6
|
+
from typing import Tuple
|
|
13
7
|
|
|
14
8
|
import click
|
|
15
|
-
import yaml
|
|
16
|
-
from requests import Response
|
|
17
9
|
|
|
18
|
-
from tinybird.prompts import test_create_prompt
|
|
19
10
|
from tinybird.tb.client import TinyB
|
|
20
|
-
from tinybird.tb.modules.build import process as build_project
|
|
21
11
|
from tinybird.tb.modules.cli import cli
|
|
22
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
23
|
-
from tinybird.tb.modules.exceptions import CLITestException
|
|
24
|
-
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
25
|
-
from tinybird.tb.modules.llm import LLM
|
|
26
|
-
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
27
|
-
from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
|
|
28
12
|
from tinybird.tb.modules.project import Project
|
|
29
|
-
from tinybird.tb.modules.
|
|
30
|
-
|
|
31
|
-
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def repr_str(dumper, data):
|
|
35
|
-
if "\n" in data:
|
|
36
|
-
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
|
|
37
|
-
return dumper.org_represent_str(data)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w") -> Path:
|
|
44
|
-
base = Path("tests")
|
|
45
|
-
if folder:
|
|
46
|
-
base = Path(folder) / base
|
|
47
|
-
|
|
48
|
-
base.mkdir(parents=True, exist_ok=True)
|
|
49
|
-
|
|
50
|
-
yaml_str = yaml.safe_dump(tests, sort_keys=False)
|
|
51
|
-
formatted_yaml = yaml_str.replace("- name:", "\n- name:")
|
|
52
|
-
|
|
53
|
-
path = base / f"{pipe_name}.yaml"
|
|
54
|
-
with open(path, mode) as f:
|
|
55
|
-
f.write(formatted_yaml)
|
|
56
|
-
return path
|
|
13
|
+
from tinybird.tb.modules.test_common import create_test, run_tests, update_test
|
|
57
14
|
|
|
58
15
|
|
|
59
16
|
@cli.group()
|
|
@@ -75,75 +32,9 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
75
32
|
"""
|
|
76
33
|
Create a test for an existing pipe
|
|
77
34
|
"""
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
load_secrets(project=project, client=client)
|
|
82
|
-
click.echo(FeedbackManager.highlight(message="\n» Building project"))
|
|
83
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
84
|
-
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
85
|
-
config = CLIConfig.get_project_config()
|
|
86
|
-
folder = project.folder
|
|
87
|
-
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
88
|
-
pipe_name = pipe_path.stem
|
|
89
|
-
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
90
|
-
pipe_content = pipe_path.read_text()
|
|
91
|
-
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
92
|
-
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
93
|
-
|
|
94
|
-
system_prompt = test_create_prompt.format(
|
|
95
|
-
name=pipe_name,
|
|
96
|
-
content=pipe_content,
|
|
97
|
-
parameters=parameters or "No parameters",
|
|
98
|
-
)
|
|
99
|
-
user_token = config.get_user_token()
|
|
100
|
-
if not user_token:
|
|
101
|
-
raise Exception("No user token found")
|
|
102
|
-
|
|
103
|
-
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
104
|
-
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
105
|
-
response_xml = extract_xml(response_llm, "response")
|
|
106
|
-
tests_content = parse_xml(response_xml, "test")
|
|
107
|
-
|
|
108
|
-
tests: List[Dict[str, Any]] = []
|
|
109
|
-
|
|
110
|
-
for test_content in tests_content:
|
|
111
|
-
test: Dict[str, Any] = {}
|
|
112
|
-
test["name"] = extract_xml(test_content, "name")
|
|
113
|
-
test["description"] = extract_xml(test_content, "description")
|
|
114
|
-
parameters_api = extract_xml(test_content, "parameters")
|
|
115
|
-
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
116
|
-
test["expected_result"] = ""
|
|
117
|
-
|
|
118
|
-
response = None
|
|
119
|
-
try:
|
|
120
|
-
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
121
|
-
except Exception:
|
|
122
|
-
pass
|
|
123
|
-
|
|
124
|
-
if response:
|
|
125
|
-
if response.status_code >= 400:
|
|
126
|
-
test["expected_http_status"] = response.status_code
|
|
127
|
-
test["expected_result"] = response.json()["error"]
|
|
128
|
-
else:
|
|
129
|
-
test.pop("expected_http_status", None)
|
|
130
|
-
test["expected_result"] = response.text or ""
|
|
131
|
-
|
|
132
|
-
tests.append(test)
|
|
133
|
-
|
|
134
|
-
if len(tests) > 0:
|
|
135
|
-
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
136
|
-
for test in tests:
|
|
137
|
-
test_name = test["name"]
|
|
138
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
139
|
-
else:
|
|
140
|
-
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
141
|
-
|
|
142
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
143
|
-
except Exception as e:
|
|
144
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
145
|
-
finally:
|
|
146
|
-
cleanup_test_workspace(client, project.folder)
|
|
35
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
|
+
create_test(name_or_filename, prompt, project, client)
|
|
147
38
|
|
|
148
39
|
|
|
149
40
|
@test.command(
|
|
@@ -153,51 +44,9 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
153
44
|
@click.argument("pipe", type=str)
|
|
154
45
|
@click.pass_context
|
|
155
46
|
def test_update(ctx: click.Context, pipe: str) -> None:
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
folder = project.folder
|
|
160
|
-
load_secrets(project=project, client=client)
|
|
161
|
-
click.echo(FeedbackManager.highlight(message="\n» Building project"))
|
|
162
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
163
|
-
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
164
|
-
pipe_tests_path = get_pipe_path(pipe, folder)
|
|
165
|
-
pipe_name = pipe_tests_path.stem
|
|
166
|
-
if pipe_tests_path.suffix == ".yaml":
|
|
167
|
-
pipe_name = pipe_tests_path.stem
|
|
168
|
-
else:
|
|
169
|
-
pipe_tests_path = Path("tests", f"{pipe_name}.yaml")
|
|
170
|
-
|
|
171
|
-
click.echo(FeedbackManager.highlight(message=f"\n» Updating tests expectations for {pipe_name} endpoint..."))
|
|
172
|
-
pipe_tests_path = Path(project.folder) / pipe_tests_path
|
|
173
|
-
pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
|
|
174
|
-
for test in pipe_tests_content:
|
|
175
|
-
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
176
|
-
response = None
|
|
177
|
-
try:
|
|
178
|
-
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test_params)
|
|
179
|
-
except Exception:
|
|
180
|
-
continue
|
|
181
|
-
|
|
182
|
-
if response.status_code >= 400:
|
|
183
|
-
test["expected_http_status"] = response.status_code
|
|
184
|
-
test["expected_result"] = response.json()["error"]
|
|
185
|
-
else:
|
|
186
|
-
if "expected_http_status" in test:
|
|
187
|
-
del test["expected_http_status"]
|
|
188
|
-
|
|
189
|
-
test["expected_result"] = response.text or ""
|
|
190
|
-
|
|
191
|
-
generate_test_file(pipe_name, pipe_tests_content, folder)
|
|
192
|
-
for test in pipe_tests_content:
|
|
193
|
-
test_name = test["name"]
|
|
194
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} updated"))
|
|
195
|
-
|
|
196
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
197
|
-
except Exception as e:
|
|
198
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
199
|
-
finally:
|
|
200
|
-
cleanup_test_workspace(client, project.folder)
|
|
47
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
48
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
49
|
+
update_test(pipe, project, client)
|
|
201
50
|
|
|
202
51
|
|
|
203
52
|
@test.command(
|
|
@@ -206,117 +55,7 @@ def test_update(ctx: click.Context, pipe: str) -> None:
|
|
|
206
55
|
)
|
|
207
56
|
@click.argument("name", nargs=-1)
|
|
208
57
|
@click.pass_context
|
|
209
|
-
def
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
load_secrets(project=project, client=client)
|
|
214
|
-
click.echo(FeedbackManager.highlight(message="\n» Building project"))
|
|
215
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
216
|
-
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
217
|
-
|
|
218
|
-
click.echo(FeedbackManager.highlight(message="\n» Running tests"))
|
|
219
|
-
paths = [Path(n) for n in name]
|
|
220
|
-
endpoints = [f"{project.path}/tests/{p.stem}.yaml" for p in paths]
|
|
221
|
-
test_files: List[str] = (
|
|
222
|
-
endpoints if len(endpoints) > 0 else glob.glob(f"{project.path}/tests/**/*.y*ml", recursive=True)
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
def run_test(test_file):
|
|
226
|
-
test_file_path = Path(test_file)
|
|
227
|
-
click.echo(FeedbackManager.info(message=f"* {test_file_path.stem}{test_file_path.suffix}"))
|
|
228
|
-
test_file_content = yaml.safe_load(test_file_path.read_text())
|
|
229
|
-
|
|
230
|
-
for test in test_file_content:
|
|
231
|
-
try:
|
|
232
|
-
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
233
|
-
response = None
|
|
234
|
-
try:
|
|
235
|
-
response = get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
236
|
-
except Exception:
|
|
237
|
-
continue
|
|
238
|
-
|
|
239
|
-
expected_result = response.text
|
|
240
|
-
if response.status_code >= 400:
|
|
241
|
-
expected_result = response.json()["error"]
|
|
242
|
-
if "expected_http_status" not in test:
|
|
243
|
-
raise Exception("Expected to not fail but got an error")
|
|
244
|
-
if test["expected_http_status"] != response.status_code:
|
|
245
|
-
raise Exception(f"Expected {test['expected_http_status']} but got {response.status_code}")
|
|
246
|
-
|
|
247
|
-
if test["expected_result"] != expected_result:
|
|
248
|
-
diff = difflib.ndiff(
|
|
249
|
-
test["expected_result"].splitlines(keepends=True), expected_result.splitlines(keepends=True)
|
|
250
|
-
)
|
|
251
|
-
printable_diff = "".join(diff)
|
|
252
|
-
raise Exception(
|
|
253
|
-
f"\nExpected: \n{test['expected_result']}\nGot: \n{expected_result}\nDiff: \n{printable_diff}"
|
|
254
|
-
)
|
|
255
|
-
click.echo(FeedbackManager.info(message=f"✓ {test['name']} passed"))
|
|
256
|
-
except Exception as e:
|
|
257
|
-
click.echo(FeedbackManager.error(message=f"✗ {test['name']} failed"))
|
|
258
|
-
click.echo(FeedbackManager.error(message=f"\n** Output and expected output are different: \n{e}"))
|
|
259
|
-
return False
|
|
260
|
-
return True
|
|
261
|
-
|
|
262
|
-
failed_tests_count = 0
|
|
263
|
-
test_count = len(test_files)
|
|
264
|
-
|
|
265
|
-
for test_file in test_files:
|
|
266
|
-
if not run_test(test_file):
|
|
267
|
-
failed_tests_count += 1
|
|
268
|
-
|
|
269
|
-
if failed_tests_count:
|
|
270
|
-
error = f"\n✗ {test_count - failed_tests_count}/{test_count} passed"
|
|
271
|
-
click.echo(FeedbackManager.error(message=error))
|
|
272
|
-
sys.exit(1)
|
|
273
|
-
else:
|
|
274
|
-
click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
|
|
275
|
-
except Exception as e:
|
|
276
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
277
|
-
finally:
|
|
278
|
-
cleanup_test_workspace(client, project.folder)
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
|
|
282
|
-
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
283
|
-
output_node = next(
|
|
284
|
-
(node for node in pipe["nodes"] if node["node_type"] != "default" and node["node_type"] != "standard"),
|
|
285
|
-
{"name": "not_found"},
|
|
286
|
-
)
|
|
287
|
-
if output_node["node_type"] == "endpoint":
|
|
288
|
-
return client._req_raw(f"/v0/pipes/{pipe_name}.ndjson?{test_params}")
|
|
289
|
-
|
|
290
|
-
params = {
|
|
291
|
-
"q": output_node["sql"],
|
|
292
|
-
"pipeline": pipe_name,
|
|
293
|
-
}
|
|
294
|
-
return client._req_raw(f"""/v0/sql?{urllib.parse.urlencode(params)}&{test_params}""")
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
def get_pipe_path(name_or_filename: str, folder: str) -> Path:
|
|
298
|
-
pipe_path: Optional[Path] = None
|
|
299
|
-
|
|
300
|
-
if ".pipe" in name_or_filename:
|
|
301
|
-
pipe_path = Path(name_or_filename)
|
|
302
|
-
if not pipe_path.exists():
|
|
303
|
-
pipe_path = None
|
|
304
|
-
else:
|
|
305
|
-
pipes = glob.glob(f"{folder}/**/{name_or_filename}.pipe", recursive=True)
|
|
306
|
-
pipe_path = next((Path(p) for p in pipes if Path(p).exists()), None)
|
|
307
|
-
|
|
308
|
-
if not pipe_path:
|
|
309
|
-
raise Exception(f"Pipe {name_or_filename} not found")
|
|
310
|
-
|
|
311
|
-
return pipe_path
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
def cleanup_test_workspace(client: TinyB, path: str) -> None:
|
|
315
|
-
user_client = deepcopy(client)
|
|
316
|
-
tokens = get_local_tokens()
|
|
317
|
-
try:
|
|
318
|
-
user_token = tokens["user_token"]
|
|
319
|
-
user_client.token = user_token
|
|
320
|
-
user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
|
|
321
|
-
except Exception:
|
|
322
|
-
pass
|
|
58
|
+
def run_tests_command(ctx: click.Context, name: Tuple[str, ...]) -> None:
|
|
59
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
60
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
61
|
+
run_tests(name, project, client)
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
# This is a command file for our CLI. Please keep it clean.
|
|
2
|
+
#
|
|
3
|
+
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
|
+
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
|
+
|
|
6
|
+
import difflib
|
|
7
|
+
import glob
|
|
8
|
+
import urllib.parse
|
|
9
|
+
from copy import deepcopy
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
import yaml
|
|
15
|
+
from requests import Response
|
|
16
|
+
|
|
17
|
+
from tinybird.prompts import test_create_prompt
|
|
18
|
+
from tinybird.tb.client import TinyB
|
|
19
|
+
from tinybird.tb.modules.build_common import process as build_project
|
|
20
|
+
from tinybird.tb.modules.common import sys_exit
|
|
21
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
22
|
+
from tinybird.tb.modules.exceptions import CLITestException
|
|
23
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
24
|
+
from tinybird.tb.modules.llm import LLM
|
|
25
|
+
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
26
|
+
from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
|
|
27
|
+
from tinybird.tb.modules.project import Project
|
|
28
|
+
from tinybird.tb.modules.secret_common import load_secrets
|
|
29
|
+
|
|
30
|
+
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def repr_str(dumper, data):
|
|
34
|
+
if "\n" in data:
|
|
35
|
+
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
|
|
36
|
+
return dumper.org_represent_str(data)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w") -> Path:
|
|
43
|
+
base = Path("tests")
|
|
44
|
+
if folder:
|
|
45
|
+
base = Path(folder) / base
|
|
46
|
+
|
|
47
|
+
base.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
|
|
49
|
+
yaml_str = yaml.safe_dump(tests, sort_keys=False)
|
|
50
|
+
formatted_yaml = yaml_str.replace("- name:", "\n- name:")
|
|
51
|
+
|
|
52
|
+
path = base / f"{pipe_name}.yaml"
|
|
53
|
+
with open(path, mode) as f:
|
|
54
|
+
f.write(formatted_yaml)
|
|
55
|
+
return path
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def create_test(
|
|
59
|
+
name_or_filename: str, prompt: str, project: Project, client: TinyB, preview: bool = False
|
|
60
|
+
) -> list[dict[str, Any]]:
|
|
61
|
+
"""
|
|
62
|
+
Create a test for an existing pipe
|
|
63
|
+
"""
|
|
64
|
+
tests: List[Dict[str, Any]] = []
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
load_secrets(project=project, client=client)
|
|
68
|
+
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
69
|
+
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
70
|
+
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
71
|
+
config = CLIConfig.get_project_config()
|
|
72
|
+
folder = project.folder
|
|
73
|
+
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
74
|
+
pipe_name = pipe_path.stem
|
|
75
|
+
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
76
|
+
pipe_content = pipe_path.read_text()
|
|
77
|
+
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
78
|
+
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
79
|
+
|
|
80
|
+
system_prompt = test_create_prompt.format(
|
|
81
|
+
name=pipe_name,
|
|
82
|
+
content=pipe_content,
|
|
83
|
+
parameters=parameters or "No parameters",
|
|
84
|
+
)
|
|
85
|
+
user_token = config.get_user_token()
|
|
86
|
+
if not user_token:
|
|
87
|
+
raise Exception("No user token found")
|
|
88
|
+
|
|
89
|
+
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
90
|
+
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
91
|
+
response_xml = extract_xml(response_llm, "response")
|
|
92
|
+
tests_content = parse_xml(response_xml, "test")
|
|
93
|
+
|
|
94
|
+
for test_content in tests_content:
|
|
95
|
+
test: Dict[str, Any] = {}
|
|
96
|
+
test["name"] = extract_xml(test_content, "name")
|
|
97
|
+
test["description"] = extract_xml(test_content, "description")
|
|
98
|
+
parameters_api = extract_xml(test_content, "parameters")
|
|
99
|
+
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
100
|
+
test["expected_result"] = ""
|
|
101
|
+
|
|
102
|
+
response = None
|
|
103
|
+
try:
|
|
104
|
+
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
105
|
+
except Exception:
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
if response:
|
|
109
|
+
if response.status_code >= 400:
|
|
110
|
+
test["expected_http_status"] = response.status_code
|
|
111
|
+
test["expected_result"] = response.json()["error"]
|
|
112
|
+
else:
|
|
113
|
+
test.pop("expected_http_status", None)
|
|
114
|
+
test["expected_result"] = response.text or ""
|
|
115
|
+
|
|
116
|
+
tests.append(test)
|
|
117
|
+
|
|
118
|
+
if not preview:
|
|
119
|
+
if len(tests) > 0:
|
|
120
|
+
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
121
|
+
for test in tests:
|
|
122
|
+
test_name = test["name"]
|
|
123
|
+
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
124
|
+
else:
|
|
125
|
+
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
126
|
+
|
|
127
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
128
|
+
except Exception as e:
|
|
129
|
+
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
130
|
+
finally:
|
|
131
|
+
cleanup_test_workspace(client, project.folder)
|
|
132
|
+
|
|
133
|
+
return tests
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def update_test(pipe: str, project: Project, client: TinyB) -> None:
|
|
137
|
+
try:
|
|
138
|
+
folder = project.folder
|
|
139
|
+
load_secrets(project=project, client=client)
|
|
140
|
+
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
141
|
+
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
142
|
+
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
143
|
+
pipe_tests_path = get_pipe_path(pipe, folder)
|
|
144
|
+
pipe_name = pipe_tests_path.stem
|
|
145
|
+
if pipe_tests_path.suffix == ".yaml":
|
|
146
|
+
pipe_name = pipe_tests_path.stem
|
|
147
|
+
else:
|
|
148
|
+
pipe_tests_path = Path("tests", f"{pipe_name}.yaml")
|
|
149
|
+
|
|
150
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Updating tests expectations for {pipe_name} endpoint..."))
|
|
151
|
+
pipe_tests_path = Path(project.folder) / pipe_tests_path
|
|
152
|
+
pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
|
|
153
|
+
for test in pipe_tests_content:
|
|
154
|
+
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
155
|
+
response = None
|
|
156
|
+
try:
|
|
157
|
+
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test_params)
|
|
158
|
+
except Exception:
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
if response.status_code >= 400:
|
|
162
|
+
test["expected_http_status"] = response.status_code
|
|
163
|
+
test["expected_result"] = response.json()["error"]
|
|
164
|
+
else:
|
|
165
|
+
if "expected_http_status" in test:
|
|
166
|
+
del test["expected_http_status"]
|
|
167
|
+
|
|
168
|
+
test["expected_result"] = response.text or ""
|
|
169
|
+
|
|
170
|
+
generate_test_file(pipe_name, pipe_tests_content, folder)
|
|
171
|
+
for test in pipe_tests_content:
|
|
172
|
+
test_name = test["name"]
|
|
173
|
+
click.echo(FeedbackManager.info(message=f"✓ {test_name} updated"))
|
|
174
|
+
|
|
175
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
176
|
+
except Exception as e:
|
|
177
|
+
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
178
|
+
finally:
|
|
179
|
+
cleanup_test_workspace(client, project.folder)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def run_tests(name: Tuple[str, ...], project: Project, client: TinyB) -> None:
|
|
183
|
+
full_error = ""
|
|
184
|
+
try:
|
|
185
|
+
load_secrets(project=project, client=client)
|
|
186
|
+
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
187
|
+
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
188
|
+
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
189
|
+
|
|
190
|
+
click.echo(FeedbackManager.highlight(message="\n» Running tests"))
|
|
191
|
+
paths = [Path(n) for n in name]
|
|
192
|
+
endpoints = [f"{project.path}/tests/{p.stem}.yaml" for p in paths]
|
|
193
|
+
test_files: List[str] = (
|
|
194
|
+
endpoints if len(endpoints) > 0 else glob.glob(f"{project.path}/tests/**/*.y*ml", recursive=True)
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
def run_test(test_file) -> Optional[str]:
|
|
198
|
+
test_file_path = Path(test_file)
|
|
199
|
+
click.echo(FeedbackManager.info(message=f"* {test_file_path.stem}{test_file_path.suffix}"))
|
|
200
|
+
test_file_content = yaml.safe_load(test_file_path.read_text())
|
|
201
|
+
test_file_errors = ""
|
|
202
|
+
for test in test_file_content:
|
|
203
|
+
try:
|
|
204
|
+
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
205
|
+
response = None
|
|
206
|
+
try:
|
|
207
|
+
response = get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
208
|
+
except Exception:
|
|
209
|
+
continue
|
|
210
|
+
|
|
211
|
+
expected_result = response.text
|
|
212
|
+
if response.status_code >= 400:
|
|
213
|
+
expected_result = response.json()["error"]
|
|
214
|
+
if "expected_http_status" not in test:
|
|
215
|
+
raise Exception("Expected to not fail but got an error")
|
|
216
|
+
if test["expected_http_status"] != response.status_code:
|
|
217
|
+
raise Exception(f"Expected {test['expected_http_status']} but got {response.status_code}")
|
|
218
|
+
|
|
219
|
+
if test["expected_result"] != expected_result:
|
|
220
|
+
diff = difflib.ndiff(
|
|
221
|
+
test["expected_result"].splitlines(keepends=True), expected_result.splitlines(keepends=True)
|
|
222
|
+
)
|
|
223
|
+
printable_diff = "".join(diff)
|
|
224
|
+
raise Exception(
|
|
225
|
+
f"\nExpected: \n{test['expected_result']}\nGot: \n{expected_result}\nDiff: \n{printable_diff}"
|
|
226
|
+
)
|
|
227
|
+
click.echo(FeedbackManager.info(message=f"✓ {test['name']} passed"))
|
|
228
|
+
except Exception as e:
|
|
229
|
+
test_file_errors += f"✗ {test['name']} failed\n** Output and expected output are different: \n{e}"
|
|
230
|
+
click.echo(FeedbackManager.error(message=test_file_errors))
|
|
231
|
+
return test_file_errors
|
|
232
|
+
return None
|
|
233
|
+
|
|
234
|
+
failed_tests_count = 0
|
|
235
|
+
test_count = len(test_files)
|
|
236
|
+
|
|
237
|
+
for test_file in test_files:
|
|
238
|
+
if run_test_error := run_test(test_file):
|
|
239
|
+
full_error += f"\n{run_test_error}"
|
|
240
|
+
failed_tests_count += 1
|
|
241
|
+
|
|
242
|
+
if failed_tests_count:
|
|
243
|
+
error = f"\n✗ {test_count - failed_tests_count}/{test_count} passed"
|
|
244
|
+
click.echo(FeedbackManager.error(message=error))
|
|
245
|
+
sys_exit("test_error", full_error)
|
|
246
|
+
else:
|
|
247
|
+
click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
|
|
248
|
+
except Exception as e:
|
|
249
|
+
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
250
|
+
finally:
|
|
251
|
+
cleanup_test_workspace(client, project.folder)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
|
|
255
|
+
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
256
|
+
output_node = next(
|
|
257
|
+
(node for node in pipe["nodes"] if node["node_type"] != "default" and node["node_type"] != "standard"),
|
|
258
|
+
{"name": "not_found"},
|
|
259
|
+
)
|
|
260
|
+
if output_node["node_type"] == "endpoint":
|
|
261
|
+
return client._req_raw(f"/v0/pipes/{pipe_name}.ndjson?{test_params}")
|
|
262
|
+
|
|
263
|
+
params = {
|
|
264
|
+
"q": output_node["sql"],
|
|
265
|
+
"pipeline": pipe_name,
|
|
266
|
+
}
|
|
267
|
+
return client._req_raw(f"""/v0/sql?{urllib.parse.urlencode(params)}&{test_params}""")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def get_pipe_path(name_or_filename: str, folder: str) -> Path:
|
|
271
|
+
pipe_path: Optional[Path] = None
|
|
272
|
+
|
|
273
|
+
if ".pipe" in name_or_filename:
|
|
274
|
+
pipe_path = Path(name_or_filename)
|
|
275
|
+
if not pipe_path.exists():
|
|
276
|
+
pipe_path = None
|
|
277
|
+
else:
|
|
278
|
+
pipes = glob.glob(f"{folder}/**/{name_or_filename}.pipe", recursive=True)
|
|
279
|
+
pipe_path = next((Path(p) for p in pipes if Path(p).exists()), None)
|
|
280
|
+
|
|
281
|
+
if not pipe_path:
|
|
282
|
+
raise Exception(f"Pipe {name_or_filename} not found")
|
|
283
|
+
|
|
284
|
+
return pipe_path
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def cleanup_test_workspace(client: TinyB, path: str) -> None:
|
|
288
|
+
user_client = deepcopy(client)
|
|
289
|
+
tokens = get_local_tokens()
|
|
290
|
+
try:
|
|
291
|
+
user_token = tokens["user_token"]
|
|
292
|
+
user_client.token = user_token
|
|
293
|
+
user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
|
|
294
|
+
except Exception:
|
|
295
|
+
pass
|
|
@@ -3,7 +3,7 @@ tinybird/context.py,sha256=FfqYfrGX_I7PKGTQo93utaKPDNVYWelg4Hsp3evX5wM,1291
|
|
|
3
3
|
tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
|
|
4
4
|
tinybird/feedback_manager.py,sha256=1INQFfRfuMCb9lfB8KNf4r6qC2khW568hoHjtk-wshI,69305
|
|
5
5
|
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
6
|
-
tinybird/prompts.py,sha256=
|
|
6
|
+
tinybird/prompts.py,sha256=HoDv9TxPiP8v2XoGTWYxP133dK9CEbXVv4XE5IT339c,45483
|
|
7
7
|
tinybird/sql.py,sha256=BufnOgclQokDyihtuXesOwHBsebN6wRXIxO5wKRkOwE,48299
|
|
8
8
|
tinybird/sql_template.py,sha256=AezE1o6_BzbHFi0J9OIqTrXQ5WvoX5eNVq4QCbFjGcs,100338
|
|
9
9
|
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
@@ -17,7 +17,7 @@ tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1w
|
|
|
17
17
|
tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
|
|
18
18
|
tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
|
|
19
19
|
tinybird/datafile/parse_pipe.py,sha256=xf4m0Tw44QWJzHzAm7Z7FwUoUUtr7noMYjU1NiWnX0k,3880
|
|
20
|
-
tinybird/tb/__cli__.py,sha256=
|
|
20
|
+
tinybird/tb/__cli__.py,sha256=D0G5HjONd9vfaiba7rVUC32Pic_cEL9FIR6-Z3uB_2c,247
|
|
21
21
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
22
22
|
tinybird/tb/cli.py,sha256=FdDFEIayjmsZEVsVSSvRiVYn_FHOVg_zWQzchnzfWho,1008
|
|
23
23
|
tinybird/tb/client.py,sha256=pJbdkWMXGAqKseNAvdsRRnl_c7I-DCMB0dWCQnG82nU,54146
|
|
@@ -54,7 +54,7 @@ tinybird/tb/modules/mock.py,sha256=ET8sRpmXnQsd2sSJXH_KCdREU1_XQgkORru6T357Akc,3
|
|
|
54
54
|
tinybird/tb/modules/mock_common.py,sha256=72yKp--Zo40hrycUtiajSRW2BojOsgOZFqUorQ_KQ3w,2279
|
|
55
55
|
tinybird/tb/modules/open.py,sha256=LYiuO8Z1I9O_v6pv58qpUCWFD6BT00BdeO21fRa4I4Y,1315
|
|
56
56
|
tinybird/tb/modules/pipe.py,sha256=xPKtezhnWZ6k_g82r4XpgKslofhuIxb_PvynH4gdUzI,2393
|
|
57
|
-
tinybird/tb/modules/project.py,sha256=
|
|
57
|
+
tinybird/tb/modules/project.py,sha256=503rZ9bcumFG9F8rlJXqvceO-BjU3hS6lkq8R2ZwSpg,6617
|
|
58
58
|
tinybird/tb/modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
59
59
|
tinybird/tb/modules/secret.py,sha256=9BIdh2PZDAbY2wRbf4ZDvkEltygztz1RMxgDmY1D0LI,3521
|
|
60
60
|
tinybird/tb/modules/secret_common.py,sha256=HyCLAI9WniDLwfK6SAb7ZUWorWjtf8j_GghlaTaos_I,1829
|
|
@@ -62,19 +62,20 @@ tinybird/tb/modules/shell.py,sha256=_9PaKkkh6ZjkixVtKNAtoCPqXMXMn1aQJM_Xzirn7ZM,
|
|
|
62
62
|
tinybird/tb/modules/sink.py,sha256=dK2s__my0ePIUYrqBzhPSgdWN9rbpvP1G4dT7DJzz80,3865
|
|
63
63
|
tinybird/tb/modules/table.py,sha256=4XrtjM-N0zfNtxVkbvLDQQazno1EPXnxTyo7llivfXk,11035
|
|
64
64
|
tinybird/tb/modules/telemetry.py,sha256=T9gtsQffWqG_4hRBaUJPzOfMkPwz7mH-R6Bn1XRYViA,11482
|
|
65
|
-
tinybird/tb/modules/test.py,sha256=
|
|
65
|
+
tinybird/tb/modules/test.py,sha256=O2-mS4uMU6nPi7yWPpWzshAgOlYKiGS-tkM12pXQGMI,1906
|
|
66
|
+
tinybird/tb/modules/test_common.py,sha256=LRgR2m2AgO5BYfdhboNL6ywIrRCffXsjzSwQE7A1W2Y,12459
|
|
66
67
|
tinybird/tb/modules/token.py,sha256=DkXW9FNCLGBisXewfk195jTJ6B1Iz7zq3cEEac48aAs,12731
|
|
67
68
|
tinybird/tb/modules/watch.py,sha256=No0bK1M1_3CYuMaIgylxf7vYFJ72lTJe3brz6xQ-mJo,8819
|
|
68
69
|
tinybird/tb/modules/workspace.py,sha256=Q_8HcxMsNg8QG9aBlwcWS2umrDP5IkTIHqqz3sfmGuc,11341
|
|
69
70
|
tinybird/tb/modules/workspace_members.py,sha256=5JdkJgfuEwbq-t6vxkBhYwgsiTDxF790wsa6Xfif9nk,8608
|
|
70
71
|
tinybird/tb/modules/agent/__init__.py,sha256=i3oe3vDIWWPaicdCM0zs7D7BJ1W0k7th93ooskHAV00,54
|
|
71
|
-
tinybird/tb/modules/agent/agent.py,sha256=
|
|
72
|
+
tinybird/tb/modules/agent/agent.py,sha256=1EGRxcIa3QTIO4EXrGAs8svBm4vlm3DiVCZEF5k7mFw,23773
|
|
72
73
|
tinybird/tb/modules/agent/animations.py,sha256=4WOC5_2BracttmMCrV0H91tXfWcUzQHBUaIJc5FA7tE,3490
|
|
73
74
|
tinybird/tb/modules/agent/banner.py,sha256=7f97PeCPW-oW9mReQn3D0by8mnDhoc0VbfebEPRPI7c,3070
|
|
74
75
|
tinybird/tb/modules/agent/memory.py,sha256=O6Kumn9AyKxcTkhI45yjAUZ3ZIAibLOcNWoiEuLYeqY,3245
|
|
75
76
|
tinybird/tb/modules/agent/models.py,sha256=LW1D27gjcd_jwFmghEzteCgToDfodX2B6B5S8BYbysw,735
|
|
76
|
-
tinybird/tb/modules/agent/prompts.py,sha256
|
|
77
|
-
tinybird/tb/modules/agent/utils.py,sha256=
|
|
77
|
+
tinybird/tb/modules/agent/prompts.py,sha256=-qks8FvYb8_WpchyjjWi9k8tQEYTjX9fEynuFrFLbzA,27727
|
|
78
|
+
tinybird/tb/modules/agent/utils.py,sha256=tVymkZdDINcp5iLPrjEWEsoGmj4SjNbvgsXcYqsMrhU,26888
|
|
78
79
|
tinybird/tb/modules/agent/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
79
80
|
tinybird/tb/modules/agent/tools/analyze.py,sha256=fTKjMCq8ckvvqq8WqpE2aE_gO0XBM-eFPt75WTFC3ik,3565
|
|
80
81
|
tinybird/tb/modules/agent/tools/append.py,sha256=b8nCO788MAk1ahITdNrrzMdUDhj-Y6KBwOCj4UmCQxg,5606
|
|
@@ -91,6 +92,7 @@ tinybird/tb/modules/agent/tools/mock.py,sha256=4JJ_45uWkbDhTGeKpwDv8L07ewsbF8u7q
|
|
|
91
92
|
tinybird/tb/modules/agent/tools/plan.py,sha256=2KHLNkr2f1RfkbAR4mCVsv94LGosXd8-ky7v6BB1OtQ,985
|
|
92
93
|
tinybird/tb/modules/agent/tools/preview_datafile.py,sha256=Gbao_FxhXstnUnngVQxztpizjugyfx1rOXTkw7Yabls,858
|
|
93
94
|
tinybird/tb/modules/agent/tools/request_endpoint.py,sha256=xNYIEFI9sfq3SjevYqJaHH1EP6uEX5HHrbKS6qo3Imo,2852
|
|
95
|
+
tinybird/tb/modules/agent/tools/test.py,sha256=0gXTfx6Yj1ZPDNDkKJj0RRhA6RQr6UtTM43KsHrdR_Y,4813
|
|
94
96
|
tinybird/tb/modules/datafile/build.py,sha256=NFKBrusFLU0WJNCXePAFWiEDuTaXpwc0lHlOQWEJ43s,51117
|
|
95
97
|
tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
|
|
96
98
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=Ra8pVQBDafbFRUKlhpgohhTsRyp_ADKZJVG8Gd69idY,17227
|
|
@@ -111,8 +113,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
111
113
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
112
114
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
113
115
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
114
|
-
tinybird-0.0.1.
|
|
115
|
-
tinybird-0.0.1.
|
|
116
|
-
tinybird-0.0.1.
|
|
117
|
-
tinybird-0.0.1.
|
|
118
|
-
tinybird-0.0.1.
|
|
116
|
+
tinybird-0.0.1.dev259.dist-info/METADATA,sha256=WTSSsjGRnfYv-YUjstmzMF58_YHbuYjg8OWOoghGaVg,1733
|
|
117
|
+
tinybird-0.0.1.dev259.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
118
|
+
tinybird-0.0.1.dev259.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
119
|
+
tinybird-0.0.1.dev259.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
120
|
+
tinybird-0.0.1.dev259.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|