tinybird 0.0.1.dev257__py3-none-any.whl → 0.0.1.dev259__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/prompts.py +1 -0
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/agent/agent.py +31 -3
- tinybird/tb/modules/agent/banner.py +1 -0
- tinybird/tb/modules/agent/prompts.py +35 -3
- tinybird/tb/modules/agent/tools/analyze.py +3 -1
- tinybird/tb/modules/agent/tools/append.py +5 -9
- tinybird/tb/modules/agent/tools/build.py +2 -1
- tinybird/tb/modules/agent/tools/create_datafile.py +3 -5
- tinybird/tb/modules/agent/tools/deploy.py +2 -5
- tinybird/tb/modules/agent/tools/deploy_check.py +2 -5
- tinybird/tb/modules/agent/tools/mock.py +3 -5
- tinybird/tb/modules/agent/tools/plan.py +12 -20
- tinybird/tb/modules/agent/tools/test.py +120 -0
- tinybird/tb/modules/agent/utils.py +10 -3
- tinybird/tb/modules/project.py +24 -1
- tinybird/tb/modules/test.py +12 -273
- tinybird/tb/modules/test_common.py +295 -0
- {tinybird-0.0.1.dev257.dist-info → tinybird-0.0.1.dev259.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev257.dist-info → tinybird-0.0.1.dev259.dist-info}/RECORD +23 -21
- {tinybird-0.0.1.dev257.dist-info → tinybird-0.0.1.dev259.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev257.dist-info → tinybird-0.0.1.dev259.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev257.dist-info → tinybird-0.0.1.dev259.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/test.py
CHANGED
|
@@ -3,57 +3,14 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import
|
|
7
|
-
import glob
|
|
8
|
-
import sys
|
|
9
|
-
import urllib.parse
|
|
10
|
-
from copy import deepcopy
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
from typing import Any, Dict, List, Optional, Tuple
|
|
6
|
+
from typing import Tuple
|
|
13
7
|
|
|
14
8
|
import click
|
|
15
|
-
import yaml
|
|
16
|
-
from requests import Response
|
|
17
9
|
|
|
18
|
-
from tinybird.prompts import test_create_prompt
|
|
19
10
|
from tinybird.tb.client import TinyB
|
|
20
|
-
from tinybird.tb.modules.build import process as build_project
|
|
21
11
|
from tinybird.tb.modules.cli import cli
|
|
22
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
23
|
-
from tinybird.tb.modules.exceptions import CLITestException
|
|
24
|
-
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
25
|
-
from tinybird.tb.modules.llm import LLM
|
|
26
|
-
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
27
|
-
from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
|
|
28
12
|
from tinybird.tb.modules.project import Project
|
|
29
|
-
from tinybird.tb.modules.
|
|
30
|
-
|
|
31
|
-
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def repr_str(dumper, data):
|
|
35
|
-
if "\n" in data:
|
|
36
|
-
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
|
|
37
|
-
return dumper.org_represent_str(data)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w") -> Path:
|
|
44
|
-
base = Path("tests")
|
|
45
|
-
if folder:
|
|
46
|
-
base = Path(folder) / base
|
|
47
|
-
|
|
48
|
-
base.mkdir(parents=True, exist_ok=True)
|
|
49
|
-
|
|
50
|
-
yaml_str = yaml.safe_dump(tests, sort_keys=False)
|
|
51
|
-
formatted_yaml = yaml_str.replace("- name:", "\n- name:")
|
|
52
|
-
|
|
53
|
-
path = base / f"{pipe_name}.yaml"
|
|
54
|
-
with open(path, mode) as f:
|
|
55
|
-
f.write(formatted_yaml)
|
|
56
|
-
return path
|
|
13
|
+
from tinybird.tb.modules.test_common import create_test, run_tests, update_test
|
|
57
14
|
|
|
58
15
|
|
|
59
16
|
@cli.group()
|
|
@@ -75,75 +32,9 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
75
32
|
"""
|
|
76
33
|
Create a test for an existing pipe
|
|
77
34
|
"""
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
load_secrets(project=project, client=client)
|
|
82
|
-
click.echo(FeedbackManager.highlight(message="\n» Building project"))
|
|
83
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
84
|
-
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
85
|
-
config = CLIConfig.get_project_config()
|
|
86
|
-
folder = project.folder
|
|
87
|
-
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
88
|
-
pipe_name = pipe_path.stem
|
|
89
|
-
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
90
|
-
pipe_content = pipe_path.read_text()
|
|
91
|
-
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
92
|
-
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
93
|
-
|
|
94
|
-
system_prompt = test_create_prompt.format(
|
|
95
|
-
name=pipe_name,
|
|
96
|
-
content=pipe_content,
|
|
97
|
-
parameters=parameters or "No parameters",
|
|
98
|
-
)
|
|
99
|
-
user_token = config.get_user_token()
|
|
100
|
-
if not user_token:
|
|
101
|
-
raise Exception("No user token found")
|
|
102
|
-
|
|
103
|
-
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
104
|
-
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
105
|
-
response_xml = extract_xml(response_llm, "response")
|
|
106
|
-
tests_content = parse_xml(response_xml, "test")
|
|
107
|
-
|
|
108
|
-
tests: List[Dict[str, Any]] = []
|
|
109
|
-
|
|
110
|
-
for test_content in tests_content:
|
|
111
|
-
test: Dict[str, Any] = {}
|
|
112
|
-
test["name"] = extract_xml(test_content, "name")
|
|
113
|
-
test["description"] = extract_xml(test_content, "description")
|
|
114
|
-
parameters_api = extract_xml(test_content, "parameters")
|
|
115
|
-
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
116
|
-
test["expected_result"] = ""
|
|
117
|
-
|
|
118
|
-
response = None
|
|
119
|
-
try:
|
|
120
|
-
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
121
|
-
except Exception:
|
|
122
|
-
pass
|
|
123
|
-
|
|
124
|
-
if response:
|
|
125
|
-
if response.status_code >= 400:
|
|
126
|
-
test["expected_http_status"] = response.status_code
|
|
127
|
-
test["expected_result"] = response.json()["error"]
|
|
128
|
-
else:
|
|
129
|
-
test.pop("expected_http_status", None)
|
|
130
|
-
test["expected_result"] = response.text or ""
|
|
131
|
-
|
|
132
|
-
tests.append(test)
|
|
133
|
-
|
|
134
|
-
if len(tests) > 0:
|
|
135
|
-
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
136
|
-
for test in tests:
|
|
137
|
-
test_name = test["name"]
|
|
138
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
139
|
-
else:
|
|
140
|
-
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
141
|
-
|
|
142
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
143
|
-
except Exception as e:
|
|
144
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
145
|
-
finally:
|
|
146
|
-
cleanup_test_workspace(client, project.folder)
|
|
35
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
|
+
create_test(name_or_filename, prompt, project, client)
|
|
147
38
|
|
|
148
39
|
|
|
149
40
|
@test.command(
|
|
@@ -153,51 +44,9 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
153
44
|
@click.argument("pipe", type=str)
|
|
154
45
|
@click.pass_context
|
|
155
46
|
def test_update(ctx: click.Context, pipe: str) -> None:
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
folder = project.folder
|
|
160
|
-
load_secrets(project=project, client=client)
|
|
161
|
-
click.echo(FeedbackManager.highlight(message="\n» Building project"))
|
|
162
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
163
|
-
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
164
|
-
pipe_tests_path = get_pipe_path(pipe, folder)
|
|
165
|
-
pipe_name = pipe_tests_path.stem
|
|
166
|
-
if pipe_tests_path.suffix == ".yaml":
|
|
167
|
-
pipe_name = pipe_tests_path.stem
|
|
168
|
-
else:
|
|
169
|
-
pipe_tests_path = Path("tests", f"{pipe_name}.yaml")
|
|
170
|
-
|
|
171
|
-
click.echo(FeedbackManager.highlight(message=f"\n» Updating tests expectations for {pipe_name} endpoint..."))
|
|
172
|
-
pipe_tests_path = Path(project.folder) / pipe_tests_path
|
|
173
|
-
pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
|
|
174
|
-
for test in pipe_tests_content:
|
|
175
|
-
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
176
|
-
response = None
|
|
177
|
-
try:
|
|
178
|
-
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test_params)
|
|
179
|
-
except Exception:
|
|
180
|
-
continue
|
|
181
|
-
|
|
182
|
-
if response.status_code >= 400:
|
|
183
|
-
test["expected_http_status"] = response.status_code
|
|
184
|
-
test["expected_result"] = response.json()["error"]
|
|
185
|
-
else:
|
|
186
|
-
if "expected_http_status" in test:
|
|
187
|
-
del test["expected_http_status"]
|
|
188
|
-
|
|
189
|
-
test["expected_result"] = response.text or ""
|
|
190
|
-
|
|
191
|
-
generate_test_file(pipe_name, pipe_tests_content, folder)
|
|
192
|
-
for test in pipe_tests_content:
|
|
193
|
-
test_name = test["name"]
|
|
194
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} updated"))
|
|
195
|
-
|
|
196
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
197
|
-
except Exception as e:
|
|
198
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
199
|
-
finally:
|
|
200
|
-
cleanup_test_workspace(client, project.folder)
|
|
47
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
48
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
49
|
+
update_test(pipe, project, client)
|
|
201
50
|
|
|
202
51
|
|
|
203
52
|
@test.command(
|
|
@@ -206,117 +55,7 @@ def test_update(ctx: click.Context, pipe: str) -> None:
|
|
|
206
55
|
)
|
|
207
56
|
@click.argument("name", nargs=-1)
|
|
208
57
|
@click.pass_context
|
|
209
|
-
def
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
load_secrets(project=project, client=client)
|
|
214
|
-
click.echo(FeedbackManager.highlight(message="\n» Building project"))
|
|
215
|
-
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
216
|
-
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
217
|
-
|
|
218
|
-
click.echo(FeedbackManager.highlight(message="\n» Running tests"))
|
|
219
|
-
paths = [Path(n) for n in name]
|
|
220
|
-
endpoints = [f"{project.path}/tests/{p.stem}.yaml" for p in paths]
|
|
221
|
-
test_files: List[str] = (
|
|
222
|
-
endpoints if len(endpoints) > 0 else glob.glob(f"{project.path}/tests/**/*.y*ml", recursive=True)
|
|
223
|
-
)
|
|
224
|
-
|
|
225
|
-
def run_test(test_file):
|
|
226
|
-
test_file_path = Path(test_file)
|
|
227
|
-
click.echo(FeedbackManager.info(message=f"* {test_file_path.stem}{test_file_path.suffix}"))
|
|
228
|
-
test_file_content = yaml.safe_load(test_file_path.read_text())
|
|
229
|
-
|
|
230
|
-
for test in test_file_content:
|
|
231
|
-
try:
|
|
232
|
-
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
233
|
-
response = None
|
|
234
|
-
try:
|
|
235
|
-
response = get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
236
|
-
except Exception:
|
|
237
|
-
continue
|
|
238
|
-
|
|
239
|
-
expected_result = response.text
|
|
240
|
-
if response.status_code >= 400:
|
|
241
|
-
expected_result = response.json()["error"]
|
|
242
|
-
if "expected_http_status" not in test:
|
|
243
|
-
raise Exception("Expected to not fail but got an error")
|
|
244
|
-
if test["expected_http_status"] != response.status_code:
|
|
245
|
-
raise Exception(f"Expected {test['expected_http_status']} but got {response.status_code}")
|
|
246
|
-
|
|
247
|
-
if test["expected_result"] != expected_result:
|
|
248
|
-
diff = difflib.ndiff(
|
|
249
|
-
test["expected_result"].splitlines(keepends=True), expected_result.splitlines(keepends=True)
|
|
250
|
-
)
|
|
251
|
-
printable_diff = "".join(diff)
|
|
252
|
-
raise Exception(
|
|
253
|
-
f"\nExpected: \n{test['expected_result']}\nGot: \n{expected_result}\nDiff: \n{printable_diff}"
|
|
254
|
-
)
|
|
255
|
-
click.echo(FeedbackManager.info(message=f"✓ {test['name']} passed"))
|
|
256
|
-
except Exception as e:
|
|
257
|
-
click.echo(FeedbackManager.error(message=f"✗ {test['name']} failed"))
|
|
258
|
-
click.echo(FeedbackManager.error(message=f"\n** Output and expected output are different: \n{e}"))
|
|
259
|
-
return False
|
|
260
|
-
return True
|
|
261
|
-
|
|
262
|
-
failed_tests_count = 0
|
|
263
|
-
test_count = len(test_files)
|
|
264
|
-
|
|
265
|
-
for test_file in test_files:
|
|
266
|
-
if not run_test(test_file):
|
|
267
|
-
failed_tests_count += 1
|
|
268
|
-
|
|
269
|
-
if failed_tests_count:
|
|
270
|
-
error = f"\n✗ {test_count - failed_tests_count}/{test_count} passed"
|
|
271
|
-
click.echo(FeedbackManager.error(message=error))
|
|
272
|
-
sys.exit(1)
|
|
273
|
-
else:
|
|
274
|
-
click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
|
|
275
|
-
except Exception as e:
|
|
276
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
277
|
-
finally:
|
|
278
|
-
cleanup_test_workspace(client, project.folder)
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
|
|
282
|
-
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
283
|
-
output_node = next(
|
|
284
|
-
(node for node in pipe["nodes"] if node["node_type"] != "default" and node["node_type"] != "standard"),
|
|
285
|
-
{"name": "not_found"},
|
|
286
|
-
)
|
|
287
|
-
if output_node["node_type"] == "endpoint":
|
|
288
|
-
return client._req_raw(f"/v0/pipes/{pipe_name}.ndjson?{test_params}")
|
|
289
|
-
|
|
290
|
-
params = {
|
|
291
|
-
"q": output_node["sql"],
|
|
292
|
-
"pipeline": pipe_name,
|
|
293
|
-
}
|
|
294
|
-
return client._req_raw(f"""/v0/sql?{urllib.parse.urlencode(params)}&{test_params}""")
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
def get_pipe_path(name_or_filename: str, folder: str) -> Path:
|
|
298
|
-
pipe_path: Optional[Path] = None
|
|
299
|
-
|
|
300
|
-
if ".pipe" in name_or_filename:
|
|
301
|
-
pipe_path = Path(name_or_filename)
|
|
302
|
-
if not pipe_path.exists():
|
|
303
|
-
pipe_path = None
|
|
304
|
-
else:
|
|
305
|
-
pipes = glob.glob(f"{folder}/**/{name_or_filename}.pipe", recursive=True)
|
|
306
|
-
pipe_path = next((Path(p) for p in pipes if Path(p).exists()), None)
|
|
307
|
-
|
|
308
|
-
if not pipe_path:
|
|
309
|
-
raise Exception(f"Pipe {name_or_filename} not found")
|
|
310
|
-
|
|
311
|
-
return pipe_path
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
def cleanup_test_workspace(client: TinyB, path: str) -> None:
|
|
315
|
-
user_client = deepcopy(client)
|
|
316
|
-
tokens = get_local_tokens()
|
|
317
|
-
try:
|
|
318
|
-
user_token = tokens["user_token"]
|
|
319
|
-
user_client.token = user_token
|
|
320
|
-
user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
|
|
321
|
-
except Exception:
|
|
322
|
-
pass
|
|
58
|
+
def run_tests_command(ctx: click.Context, name: Tuple[str, ...]) -> None:
|
|
59
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
60
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
61
|
+
run_tests(name, project, client)
|
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
# This is a command file for our CLI. Please keep it clean.
|
|
2
|
+
#
|
|
3
|
+
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
|
+
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
|
+
|
|
6
|
+
import difflib
|
|
7
|
+
import glob
|
|
8
|
+
import urllib.parse
|
|
9
|
+
from copy import deepcopy
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
12
|
+
|
|
13
|
+
import click
|
|
14
|
+
import yaml
|
|
15
|
+
from requests import Response
|
|
16
|
+
|
|
17
|
+
from tinybird.prompts import test_create_prompt
|
|
18
|
+
from tinybird.tb.client import TinyB
|
|
19
|
+
from tinybird.tb.modules.build_common import process as build_project
|
|
20
|
+
from tinybird.tb.modules.common import sys_exit
|
|
21
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
22
|
+
from tinybird.tb.modules.exceptions import CLITestException
|
|
23
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
24
|
+
from tinybird.tb.modules.llm import LLM
|
|
25
|
+
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
26
|
+
from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
|
|
27
|
+
from tinybird.tb.modules.project import Project
|
|
28
|
+
from tinybird.tb.modules.secret_common import load_secrets
|
|
29
|
+
|
|
30
|
+
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def repr_str(dumper, data):
|
|
34
|
+
if "\n" in data:
|
|
35
|
+
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
|
|
36
|
+
return dumper.org_represent_str(data)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w") -> Path:
|
|
43
|
+
base = Path("tests")
|
|
44
|
+
if folder:
|
|
45
|
+
base = Path(folder) / base
|
|
46
|
+
|
|
47
|
+
base.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
|
|
49
|
+
yaml_str = yaml.safe_dump(tests, sort_keys=False)
|
|
50
|
+
formatted_yaml = yaml_str.replace("- name:", "\n- name:")
|
|
51
|
+
|
|
52
|
+
path = base / f"{pipe_name}.yaml"
|
|
53
|
+
with open(path, mode) as f:
|
|
54
|
+
f.write(formatted_yaml)
|
|
55
|
+
return path
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def create_test(
|
|
59
|
+
name_or_filename: str, prompt: str, project: Project, client: TinyB, preview: bool = False
|
|
60
|
+
) -> list[dict[str, Any]]:
|
|
61
|
+
"""
|
|
62
|
+
Create a test for an existing pipe
|
|
63
|
+
"""
|
|
64
|
+
tests: List[Dict[str, Any]] = []
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
load_secrets(project=project, client=client)
|
|
68
|
+
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
69
|
+
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
70
|
+
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
71
|
+
config = CLIConfig.get_project_config()
|
|
72
|
+
folder = project.folder
|
|
73
|
+
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
74
|
+
pipe_name = pipe_path.stem
|
|
75
|
+
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
76
|
+
pipe_content = pipe_path.read_text()
|
|
77
|
+
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
78
|
+
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
79
|
+
|
|
80
|
+
system_prompt = test_create_prompt.format(
|
|
81
|
+
name=pipe_name,
|
|
82
|
+
content=pipe_content,
|
|
83
|
+
parameters=parameters or "No parameters",
|
|
84
|
+
)
|
|
85
|
+
user_token = config.get_user_token()
|
|
86
|
+
if not user_token:
|
|
87
|
+
raise Exception("No user token found")
|
|
88
|
+
|
|
89
|
+
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
90
|
+
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
91
|
+
response_xml = extract_xml(response_llm, "response")
|
|
92
|
+
tests_content = parse_xml(response_xml, "test")
|
|
93
|
+
|
|
94
|
+
for test_content in tests_content:
|
|
95
|
+
test: Dict[str, Any] = {}
|
|
96
|
+
test["name"] = extract_xml(test_content, "name")
|
|
97
|
+
test["description"] = extract_xml(test_content, "description")
|
|
98
|
+
parameters_api = extract_xml(test_content, "parameters")
|
|
99
|
+
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
100
|
+
test["expected_result"] = ""
|
|
101
|
+
|
|
102
|
+
response = None
|
|
103
|
+
try:
|
|
104
|
+
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
105
|
+
except Exception:
|
|
106
|
+
pass
|
|
107
|
+
|
|
108
|
+
if response:
|
|
109
|
+
if response.status_code >= 400:
|
|
110
|
+
test["expected_http_status"] = response.status_code
|
|
111
|
+
test["expected_result"] = response.json()["error"]
|
|
112
|
+
else:
|
|
113
|
+
test.pop("expected_http_status", None)
|
|
114
|
+
test["expected_result"] = response.text or ""
|
|
115
|
+
|
|
116
|
+
tests.append(test)
|
|
117
|
+
|
|
118
|
+
if not preview:
|
|
119
|
+
if len(tests) > 0:
|
|
120
|
+
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
121
|
+
for test in tests:
|
|
122
|
+
test_name = test["name"]
|
|
123
|
+
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
124
|
+
else:
|
|
125
|
+
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
126
|
+
|
|
127
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
128
|
+
except Exception as e:
|
|
129
|
+
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
130
|
+
finally:
|
|
131
|
+
cleanup_test_workspace(client, project.folder)
|
|
132
|
+
|
|
133
|
+
return tests
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def update_test(pipe: str, project: Project, client: TinyB) -> None:
|
|
137
|
+
try:
|
|
138
|
+
folder = project.folder
|
|
139
|
+
load_secrets(project=project, client=client)
|
|
140
|
+
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
141
|
+
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
142
|
+
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
143
|
+
pipe_tests_path = get_pipe_path(pipe, folder)
|
|
144
|
+
pipe_name = pipe_tests_path.stem
|
|
145
|
+
if pipe_tests_path.suffix == ".yaml":
|
|
146
|
+
pipe_name = pipe_tests_path.stem
|
|
147
|
+
else:
|
|
148
|
+
pipe_tests_path = Path("tests", f"{pipe_name}.yaml")
|
|
149
|
+
|
|
150
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Updating tests expectations for {pipe_name} endpoint..."))
|
|
151
|
+
pipe_tests_path = Path(project.folder) / pipe_tests_path
|
|
152
|
+
pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
|
|
153
|
+
for test in pipe_tests_content:
|
|
154
|
+
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
155
|
+
response = None
|
|
156
|
+
try:
|
|
157
|
+
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test_params)
|
|
158
|
+
except Exception:
|
|
159
|
+
continue
|
|
160
|
+
|
|
161
|
+
if response.status_code >= 400:
|
|
162
|
+
test["expected_http_status"] = response.status_code
|
|
163
|
+
test["expected_result"] = response.json()["error"]
|
|
164
|
+
else:
|
|
165
|
+
if "expected_http_status" in test:
|
|
166
|
+
del test["expected_http_status"]
|
|
167
|
+
|
|
168
|
+
test["expected_result"] = response.text or ""
|
|
169
|
+
|
|
170
|
+
generate_test_file(pipe_name, pipe_tests_content, folder)
|
|
171
|
+
for test in pipe_tests_content:
|
|
172
|
+
test_name = test["name"]
|
|
173
|
+
click.echo(FeedbackManager.info(message=f"✓ {test_name} updated"))
|
|
174
|
+
|
|
175
|
+
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
176
|
+
except Exception as e:
|
|
177
|
+
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
178
|
+
finally:
|
|
179
|
+
cleanup_test_workspace(client, project.folder)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def run_tests(name: Tuple[str, ...], project: Project, client: TinyB) -> None:
|
|
183
|
+
full_error = ""
|
|
184
|
+
try:
|
|
185
|
+
load_secrets(project=project, client=client)
|
|
186
|
+
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
187
|
+
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
188
|
+
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
189
|
+
|
|
190
|
+
click.echo(FeedbackManager.highlight(message="\n» Running tests"))
|
|
191
|
+
paths = [Path(n) for n in name]
|
|
192
|
+
endpoints = [f"{project.path}/tests/{p.stem}.yaml" for p in paths]
|
|
193
|
+
test_files: List[str] = (
|
|
194
|
+
endpoints if len(endpoints) > 0 else glob.glob(f"{project.path}/tests/**/*.y*ml", recursive=True)
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
def run_test(test_file) -> Optional[str]:
|
|
198
|
+
test_file_path = Path(test_file)
|
|
199
|
+
click.echo(FeedbackManager.info(message=f"* {test_file_path.stem}{test_file_path.suffix}"))
|
|
200
|
+
test_file_content = yaml.safe_load(test_file_path.read_text())
|
|
201
|
+
test_file_errors = ""
|
|
202
|
+
for test in test_file_content:
|
|
203
|
+
try:
|
|
204
|
+
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
205
|
+
response = None
|
|
206
|
+
try:
|
|
207
|
+
response = get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
208
|
+
except Exception:
|
|
209
|
+
continue
|
|
210
|
+
|
|
211
|
+
expected_result = response.text
|
|
212
|
+
if response.status_code >= 400:
|
|
213
|
+
expected_result = response.json()["error"]
|
|
214
|
+
if "expected_http_status" not in test:
|
|
215
|
+
raise Exception("Expected to not fail but got an error")
|
|
216
|
+
if test["expected_http_status"] != response.status_code:
|
|
217
|
+
raise Exception(f"Expected {test['expected_http_status']} but got {response.status_code}")
|
|
218
|
+
|
|
219
|
+
if test["expected_result"] != expected_result:
|
|
220
|
+
diff = difflib.ndiff(
|
|
221
|
+
test["expected_result"].splitlines(keepends=True), expected_result.splitlines(keepends=True)
|
|
222
|
+
)
|
|
223
|
+
printable_diff = "".join(diff)
|
|
224
|
+
raise Exception(
|
|
225
|
+
f"\nExpected: \n{test['expected_result']}\nGot: \n{expected_result}\nDiff: \n{printable_diff}"
|
|
226
|
+
)
|
|
227
|
+
click.echo(FeedbackManager.info(message=f"✓ {test['name']} passed"))
|
|
228
|
+
except Exception as e:
|
|
229
|
+
test_file_errors += f"✗ {test['name']} failed\n** Output and expected output are different: \n{e}"
|
|
230
|
+
click.echo(FeedbackManager.error(message=test_file_errors))
|
|
231
|
+
return test_file_errors
|
|
232
|
+
return None
|
|
233
|
+
|
|
234
|
+
failed_tests_count = 0
|
|
235
|
+
test_count = len(test_files)
|
|
236
|
+
|
|
237
|
+
for test_file in test_files:
|
|
238
|
+
if run_test_error := run_test(test_file):
|
|
239
|
+
full_error += f"\n{run_test_error}"
|
|
240
|
+
failed_tests_count += 1
|
|
241
|
+
|
|
242
|
+
if failed_tests_count:
|
|
243
|
+
error = f"\n✗ {test_count - failed_tests_count}/{test_count} passed"
|
|
244
|
+
click.echo(FeedbackManager.error(message=error))
|
|
245
|
+
sys_exit("test_error", full_error)
|
|
246
|
+
else:
|
|
247
|
+
click.echo(FeedbackManager.success(message=f"\n✓ {test_count}/{test_count} passed"))
|
|
248
|
+
except Exception as e:
|
|
249
|
+
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
250
|
+
finally:
|
|
251
|
+
cleanup_test_workspace(client, project.folder)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
|
|
255
|
+
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
256
|
+
output_node = next(
|
|
257
|
+
(node for node in pipe["nodes"] if node["node_type"] != "default" and node["node_type"] != "standard"),
|
|
258
|
+
{"name": "not_found"},
|
|
259
|
+
)
|
|
260
|
+
if output_node["node_type"] == "endpoint":
|
|
261
|
+
return client._req_raw(f"/v0/pipes/{pipe_name}.ndjson?{test_params}")
|
|
262
|
+
|
|
263
|
+
params = {
|
|
264
|
+
"q": output_node["sql"],
|
|
265
|
+
"pipeline": pipe_name,
|
|
266
|
+
}
|
|
267
|
+
return client._req_raw(f"""/v0/sql?{urllib.parse.urlencode(params)}&{test_params}""")
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def get_pipe_path(name_or_filename: str, folder: str) -> Path:
|
|
271
|
+
pipe_path: Optional[Path] = None
|
|
272
|
+
|
|
273
|
+
if ".pipe" in name_or_filename:
|
|
274
|
+
pipe_path = Path(name_or_filename)
|
|
275
|
+
if not pipe_path.exists():
|
|
276
|
+
pipe_path = None
|
|
277
|
+
else:
|
|
278
|
+
pipes = glob.glob(f"{folder}/**/{name_or_filename}.pipe", recursive=True)
|
|
279
|
+
pipe_path = next((Path(p) for p in pipes if Path(p).exists()), None)
|
|
280
|
+
|
|
281
|
+
if not pipe_path:
|
|
282
|
+
raise Exception(f"Pipe {name_or_filename} not found")
|
|
283
|
+
|
|
284
|
+
return pipe_path
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def cleanup_test_workspace(client: TinyB, path: str) -> None:
|
|
288
|
+
user_client = deepcopy(client)
|
|
289
|
+
tokens = get_local_tokens()
|
|
290
|
+
try:
|
|
291
|
+
user_token = tokens["user_token"]
|
|
292
|
+
user_client.token = user_token
|
|
293
|
+
user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
|
|
294
|
+
except Exception:
|
|
295
|
+
pass
|