tinybird 0.0.1.dev300__py3-none-any.whl → 0.0.1.dev302__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/connectors.py +1 -7
- tinybird/datafile/common.py +2 -6
- tinybird/service_datasources.py +0 -2
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/client.py +0 -4
- tinybird/tb/modules/agent/agent.py +55 -50
- tinybird/tb/modules/agent/models.py +6 -2
- tinybird/tb/modules/agent/utils.py +1 -2
- tinybird/tb/modules/build.py +2 -2
- tinybird/tb/modules/build_common.py +2 -2
- tinybird/tb/modules/cli.py +2 -3
- tinybird/tb/modules/common.py +2 -2
- tinybird/tb/modules/create.py +72 -150
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +1 -1
- tinybird/tb/modules/datasource.py +24 -6
- tinybird/tb/modules/exceptions.py +7 -0
- tinybird/tb/modules/mock.py +5 -54
- tinybird/tb/modules/mock_common.py +0 -54
- tinybird/tb/modules/shell.py +1 -1
- tinybird/tb/modules/test.py +7 -6
- tinybird/tb/modules/test_common.py +0 -89
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tornado_template.py +1 -1
- {tinybird-0.0.1.dev300.dist-info → tinybird-0.0.1.dev302.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev300.dist-info → tinybird-0.0.1.dev302.dist-info}/RECORD +29 -29
- {tinybird-0.0.1.dev300.dist-info → tinybird-0.0.1.dev302.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev300.dist-info → tinybird-0.0.1.dev302.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev300.dist-info → tinybird-0.0.1.dev302.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/create.py
CHANGED
|
@@ -3,25 +3,22 @@ import os
|
|
|
3
3
|
import re
|
|
4
4
|
import shutil
|
|
5
5
|
from pathlib import Path
|
|
6
|
-
from typing import Any, Dict, List, Optional
|
|
6
|
+
from typing import Any, Dict, List, Optional, Set
|
|
7
7
|
from urllib.parse import urlparse
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
10
|
import requests
|
|
11
11
|
|
|
12
|
-
from tinybird.prompts import claude_rules_prompt,
|
|
13
|
-
from tinybird.tb.
|
|
12
|
+
from tinybird.prompts import claude_rules_prompt, rules_prompt
|
|
13
|
+
from tinybird.tb.modules.agent import run_agent
|
|
14
14
|
from tinybird.tb.modules.cicd import init_cicd
|
|
15
15
|
from tinybird.tb.modules.cli import cli
|
|
16
|
-
from tinybird.tb.modules.common import _generate_datafile
|
|
16
|
+
from tinybird.tb.modules.common import _generate_datafile
|
|
17
17
|
from tinybird.tb.modules.config import CLIConfig
|
|
18
18
|
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
19
19
|
from tinybird.tb.modules.exceptions import CLICreateException
|
|
20
20
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
21
|
-
from tinybird.tb.modules.llm import LLM
|
|
22
|
-
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
23
21
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
24
|
-
from tinybird.tb.modules.mock_common import create_mock_data
|
|
25
22
|
from tinybird.tb.modules.project import Project
|
|
26
23
|
|
|
27
24
|
|
|
@@ -67,13 +64,9 @@ def create(
|
|
|
67
64
|
folder_path.mkdir()
|
|
68
65
|
|
|
69
66
|
try:
|
|
70
|
-
tb_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
|
|
71
|
-
user_token: str = ""
|
|
72
67
|
created_something = False
|
|
73
|
-
if prompt:
|
|
74
|
-
|
|
75
|
-
if not user_token:
|
|
76
|
-
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
68
|
+
if prompt and not ctx_config.get("user_token"):
|
|
69
|
+
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
77
70
|
|
|
78
71
|
if not validate_project_structure(project):
|
|
79
72
|
click.echo(FeedbackManager.highlight(message="\n» Creating new project structure..."))
|
|
@@ -99,28 +92,20 @@ def create(
|
|
|
99
92
|
|
|
100
93
|
prompt_result: List[Path] = []
|
|
101
94
|
if prompt:
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
if readme_path.exists():
|
|
106
|
-
click.echo(FeedbackManager.highlight(message="\n» Updating project description..."))
|
|
107
|
-
else:
|
|
108
|
-
click.echo(FeedbackManager.highlight(message="\n» Creating project description..."))
|
|
109
|
-
readme_path.touch()
|
|
110
|
-
llm = LLM(user_token=str(user_token), host=tb_client.host)
|
|
111
|
-
readme_user_prompt = prompt or ""
|
|
112
|
-
all_resources_xml = get_resources_xml(project)
|
|
113
|
-
readme_response = llm.ask(
|
|
114
|
-
system_prompt=readme_prompt(
|
|
115
|
-
readme_path.read_text(), tb_client.host, "$TB_ADMIN_TOKEN", all_resources_xml
|
|
116
|
-
),
|
|
117
|
-
prompt=readme_user_prompt,
|
|
118
|
-
feature="tb_create_readme",
|
|
95
|
+
prompt_instructions = (
|
|
96
|
+
"Create or update the Tinybird datasources, pipes, and connections required to satisfy the following request. "
|
|
97
|
+
"Do not generate mock data or append data; those steps will run later programmatically."
|
|
119
98
|
)
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
99
|
+
prompt_result = create_resources_from_prompt(
|
|
100
|
+
ctx_config,
|
|
101
|
+
project,
|
|
102
|
+
prompt,
|
|
103
|
+
feature="tb_create",
|
|
104
|
+
instructions=prompt_instructions,
|
|
105
|
+
)
|
|
106
|
+
result.extend(prompt_result)
|
|
107
|
+
if prompt_result:
|
|
108
|
+
created_something = True
|
|
124
109
|
|
|
125
110
|
if data or prompt:
|
|
126
111
|
click.echo(FeedbackManager.success(message="✓ Resources created!\n"))
|
|
@@ -180,24 +165,30 @@ def create(
|
|
|
180
165
|
datasource_name = datasource_path.stem
|
|
181
166
|
datasource_content = datasource_path.read_text()
|
|
182
167
|
has_json_path = "`json:" in datasource_content
|
|
183
|
-
if has_json_path:
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
168
|
+
if not has_json_path:
|
|
169
|
+
continue
|
|
170
|
+
|
|
171
|
+
fixture_path = Path(folder) / "fixtures" / f"{datasource_name}.ndjson"
|
|
172
|
+
fixture_existed = fixture_path.exists()
|
|
173
|
+
fixture_prompt = (
|
|
174
|
+
f"Generate {rows} rows of representative sample data for the Tinybird datasource defined in {datasource_path}. "
|
|
175
|
+
f"Store the data in ndjson format at fixtures/{datasource_name}.ndjson."
|
|
176
|
+
)
|
|
177
|
+
if prompt.strip():
|
|
178
|
+
fixture_prompt += f"\n\nOriginal project request:\n{prompt.strip()}"
|
|
179
|
+
|
|
180
|
+
run_agent(
|
|
181
|
+
ctx_config,
|
|
182
|
+
project,
|
|
183
|
+
True,
|
|
184
|
+
prompt=fixture_prompt,
|
|
185
|
+
feature="tb_mock",
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
if fixture_path.exists() and not fixture_existed:
|
|
189
|
+
click.echo(FeedbackManager.info_file_created(file=f"fixtures/{datasource_name}.ndjson"))
|
|
190
|
+
click.echo(FeedbackManager.success(message="✓ Done!"))
|
|
191
|
+
created_something = True
|
|
201
192
|
|
|
202
193
|
if not created_something and not len(result) > 0:
|
|
203
194
|
click.echo(FeedbackManager.warning(message="△ No resources created\n"))
|
|
@@ -291,86 +282,39 @@ def create_project_structure(folder: str):
|
|
|
291
282
|
|
|
292
283
|
|
|
293
284
|
def create_resources_from_prompt(
|
|
294
|
-
|
|
295
|
-
user_token: str,
|
|
296
|
-
prompt: str,
|
|
285
|
+
config: Dict[str, Any],
|
|
297
286
|
project: Project,
|
|
298
|
-
|
|
287
|
+
prompt: str,
|
|
288
|
+
feature: str = "tb_create",
|
|
289
|
+
instructions: Optional[str] = None,
|
|
299
290
|
) -> List[Path]:
|
|
300
|
-
|
|
301
|
-
datasource_paths = [Path(ds_file) for ds_file in project.get_datasource_files()]
|
|
302
|
-
pipes_paths = [Path(pipe_file) for pipe_file in project.get_pipe_files()]
|
|
303
|
-
connections_paths = [Path(conn_file) for conn_file in project.get_connection_files()]
|
|
304
|
-
resources_xml = "\n".join(
|
|
305
|
-
[
|
|
306
|
-
f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
|
|
307
|
-
for resource_type, resource_name, resource_content in [
|
|
308
|
-
("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
|
|
309
|
-
]
|
|
310
|
-
+ [
|
|
311
|
-
(
|
|
312
|
-
"pipe",
|
|
313
|
-
pipe.stem,
|
|
314
|
-
pipe.read_text(),
|
|
315
|
-
)
|
|
316
|
-
for pipe in pipes_paths
|
|
317
|
-
]
|
|
318
|
-
+ [
|
|
319
|
-
(
|
|
320
|
-
"connection",
|
|
321
|
-
conn.stem,
|
|
322
|
-
conn.read_text(),
|
|
323
|
-
)
|
|
324
|
-
for conn in connections_paths
|
|
325
|
-
]
|
|
326
|
-
]
|
|
327
|
-
)
|
|
328
|
-
llm = LLM(user_token=user_token, host=tb_client.host)
|
|
329
|
-
prompt_result = llm.ask(system_prompt=create_prompt(resources_xml), prompt=prompt, feature=feature)
|
|
330
|
-
prompt_result = extract_xml(prompt_result, "response")
|
|
331
|
-
resources = parse_xml(prompt_result, "resource")
|
|
332
|
-
datasources = []
|
|
333
|
-
pipes = []
|
|
334
|
-
connections = []
|
|
335
|
-
for resource_xml in resources:
|
|
336
|
-
resource_type = extract_xml(resource_xml, "type")
|
|
337
|
-
name = extract_xml(resource_xml, "name")
|
|
338
|
-
content = extract_xml(resource_xml, "content")
|
|
339
|
-
resource = {
|
|
340
|
-
"name": name,
|
|
341
|
-
"content": content,
|
|
342
|
-
}
|
|
343
|
-
if resource_type.lower() == "datasource":
|
|
344
|
-
datasources.append(resource)
|
|
345
|
-
elif resource_type.lower() == "pipe":
|
|
346
|
-
pipes.append(resource)
|
|
347
|
-
elif resource_type.lower() == "connection":
|
|
348
|
-
connections.append(resource)
|
|
349
|
-
|
|
350
|
-
for ds in datasources:
|
|
351
|
-
content = ds["content"].replace("```", "")
|
|
352
|
-
filename = f"{ds['name']}.datasource"
|
|
353
|
-
ds_file = generate_datafile(
|
|
354
|
-
content,
|
|
355
|
-
filename=filename,
|
|
356
|
-
data=None,
|
|
357
|
-
_format="ndjson",
|
|
358
|
-
force=True,
|
|
359
|
-
folder=project.folder,
|
|
360
|
-
)
|
|
361
|
-
result.append(ds_file)
|
|
362
|
-
for pipe in pipes:
|
|
363
|
-
content = pipe["content"].replace("```", "")
|
|
364
|
-
pipe_file = generate_pipe_file(pipe["name"], content, project.folder)
|
|
365
|
-
result.append(pipe_file)
|
|
291
|
+
"""Run the agent in prompt mode and report newly created project resources."""
|
|
366
292
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
293
|
+
agent_prompt = prompt.strip()
|
|
294
|
+
if instructions:
|
|
295
|
+
instructions = instructions.strip()
|
|
296
|
+
if agent_prompt:
|
|
297
|
+
agent_prompt = f"{instructions}\n\n{agent_prompt}"
|
|
298
|
+
else:
|
|
299
|
+
agent_prompt = instructions
|
|
372
300
|
|
|
373
|
-
|
|
301
|
+
if not agent_prompt:
|
|
302
|
+
return []
|
|
303
|
+
|
|
304
|
+
resources_before = _collect_project_resource_paths(project)
|
|
305
|
+
run_agent(config, project, True, prompt=agent_prompt, feature=feature)
|
|
306
|
+
resources_after = _collect_project_resource_paths(project)
|
|
307
|
+
|
|
308
|
+
created_resources = [Path(path) for path in sorted(resources_after - resources_before)]
|
|
309
|
+
return created_resources
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _collect_project_resource_paths(project: Project) -> Set[Path]:
|
|
313
|
+
resources: Set[Path] = set()
|
|
314
|
+
resources.update(Path(path) for path in project.get_datasource_files())
|
|
315
|
+
resources.update(Path(path) for path in project.get_pipe_files())
|
|
316
|
+
resources.update(Path(path) for path in project.get_connection_files())
|
|
317
|
+
return resources
|
|
374
318
|
|
|
375
319
|
|
|
376
320
|
def init_git(folder: str):
|
|
@@ -513,28 +457,6 @@ def save_context(prompt: str, feedback: str):
|
|
|
513
457
|
context_file.write_text(f"- {prompt}\n{feedback}")
|
|
514
458
|
|
|
515
459
|
|
|
516
|
-
def get_resources_xml(project: Project) -> str:
|
|
517
|
-
datasource_paths = [Path(f) for f in project.get_datasource_files()]
|
|
518
|
-
pipes_paths = [Path(f) for f in project.get_pipe_files()]
|
|
519
|
-
resources_xml = "\n".join(
|
|
520
|
-
[
|
|
521
|
-
f"<resource><type>{resource_type}</type><name>{resource_name}</name><content>{resource_content}</content></resource>"
|
|
522
|
-
for resource_type, resource_name, resource_content in [
|
|
523
|
-
("datasource", ds.stem, ds.read_text()) for ds in datasource_paths
|
|
524
|
-
]
|
|
525
|
-
+ [
|
|
526
|
-
(
|
|
527
|
-
"pipe",
|
|
528
|
-
pipe.stem,
|
|
529
|
-
pipe.read_text(),
|
|
530
|
-
)
|
|
531
|
-
for pipe in pipes_paths
|
|
532
|
-
]
|
|
533
|
-
]
|
|
534
|
-
)
|
|
535
|
-
return resources_xml
|
|
536
|
-
|
|
537
|
-
|
|
538
460
|
def create_resources_from_data(
|
|
539
461
|
data: str,
|
|
540
462
|
project: Project,
|
|
@@ -168,7 +168,7 @@ def diff_command(
|
|
|
168
168
|
sys.stdout.writelines(diff_lines)
|
|
169
169
|
click.echo("")
|
|
170
170
|
|
|
171
|
-
for rfilename
|
|
171
|
+
for rfilename in local_resources.keys():
|
|
172
172
|
if rfilename not in changed:
|
|
173
173
|
for resource in remote_datasources + remote_pipes:
|
|
174
174
|
properties = get_name_version(resource["name"])
|
|
@@ -768,13 +768,31 @@ def datasource_create(
|
|
|
768
768
|
|
|
769
769
|
if datasource_type == "prompt":
|
|
770
770
|
click.echo(FeedbackManager.gray(message="\n» Creating .datasource file..."))
|
|
771
|
-
|
|
772
|
-
if not user_token:
|
|
771
|
+
if not config.get("user_token"):
|
|
773
772
|
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
773
|
+
|
|
774
|
+
instructions = (
|
|
775
|
+
"Create or update a Tinybird datasource (.datasource file) for this project. "
|
|
776
|
+
"Do not generate mock data or append data; those steps will run later programmatically."
|
|
777
|
+
)
|
|
778
|
+
if name:
|
|
779
|
+
instructions += f" Name the datasource '{name}'."
|
|
780
|
+
|
|
781
|
+
created_resources = create_resources_from_prompt(
|
|
782
|
+
config,
|
|
783
|
+
project,
|
|
784
|
+
prompt,
|
|
785
|
+
feature="tb_datasource_create",
|
|
786
|
+
instructions=instructions,
|
|
787
|
+
)
|
|
788
|
+
if any(path.suffix == ".datasource" for path in created_resources):
|
|
789
|
+
click.echo(FeedbackManager.success(message="✓ .datasource created!"))
|
|
790
|
+
else:
|
|
791
|
+
click.echo(
|
|
792
|
+
FeedbackManager.gray(
|
|
793
|
+
message="△ No new datasource file detected. Existing resources may have been updated instead."
|
|
794
|
+
)
|
|
795
|
+
)
|
|
778
796
|
return
|
|
779
797
|
|
|
780
798
|
connection_required = datasource_type in ("kafka", "s3", "gcs")
|
|
@@ -127,6 +127,13 @@ class CLIMockException(CLIException):
|
|
|
127
127
|
super().__init__(message, "mock_error", **kw_telemetry_event_data)
|
|
128
128
|
|
|
129
129
|
|
|
130
|
+
class CLIAgentException(CLIException):
|
|
131
|
+
"""Exceptions generated by the agent commands"""
|
|
132
|
+
|
|
133
|
+
def __init__(self, message: str, **kw_telemetry_event_data: Any) -> None:
|
|
134
|
+
super().__init__(message, "agent_error", **kw_telemetry_event_data)
|
|
135
|
+
|
|
136
|
+
|
|
130
137
|
class CLILoginException(CLIException):
|
|
131
138
|
"""Exceptions generated by the login commands"""
|
|
132
139
|
|
tinybird/tb/modules/mock.py
CHANGED
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
import glob
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
1
|
import click
|
|
5
2
|
|
|
6
|
-
from tinybird.tb.
|
|
3
|
+
from tinybird.tb.modules.agent import run_agent
|
|
7
4
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
9
|
-
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
10
5
|
from tinybird.tb.modules.exceptions import CLIMockException
|
|
11
6
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
12
|
-
from tinybird.tb.modules.mock_common import append_mock_data, create_mock_data
|
|
13
7
|
from tinybird.tb.modules.project import Project
|
|
14
8
|
|
|
15
9
|
|
|
@@ -31,60 +25,17 @@ from tinybird.tb.modules.project import Project
|
|
|
31
25
|
)
|
|
32
26
|
@click.pass_context
|
|
33
27
|
def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
|
|
34
|
-
"""Generate sample data for a data source.
|
|
35
|
-
|
|
36
|
-
Args:
|
|
37
|
-
datasource: Path to the datasource file to load sample data into
|
|
38
|
-
rows: Number of events to send
|
|
39
|
-
prompt: Extra context to use for data generation
|
|
40
|
-
skip: Skip following up on the generated data
|
|
41
|
-
"""
|
|
28
|
+
"""Generate sample data for a data source."""
|
|
42
29
|
|
|
43
30
|
try:
|
|
44
|
-
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
45
31
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
46
32
|
ctx_config = ctx.ensure_object(dict)["config"]
|
|
33
|
+
prompt = f"""Generate mock data for the following datasource: {datasource} with {rows} rows and {format_} format. Extra context: {prompt}"""
|
|
47
34
|
env = ctx.ensure_object(dict)["env"]
|
|
48
|
-
datasource_path = Path(datasource)
|
|
49
|
-
datasource_name = datasource
|
|
50
|
-
folder = project.folder
|
|
51
|
-
click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
|
|
52
|
-
if datasource_path.suffix == ".datasource":
|
|
53
|
-
datasource_name = datasource_path.stem
|
|
54
|
-
else:
|
|
55
|
-
datasource_from_glob = glob.glob(f"{folder}/**/{datasource}.datasource")
|
|
56
|
-
if datasource_from_glob:
|
|
57
|
-
datasource_path = Path(datasource_from_glob[0])
|
|
58
|
-
|
|
59
|
-
if not datasource_path.exists():
|
|
60
|
-
raise Exception(f"Datasource '{datasource_path.stem}' not found")
|
|
61
|
-
|
|
62
|
-
datasource_content = datasource_path.read_text()
|
|
63
|
-
config = CLIConfig.get_project_config()
|
|
64
|
-
user_token = ctx_config.get("user_token")
|
|
65
|
-
|
|
66
|
-
if not user_token:
|
|
67
|
-
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
68
|
-
|
|
69
|
-
data = create_mock_data(
|
|
70
|
-
datasource_name,
|
|
71
|
-
datasource_content,
|
|
72
|
-
rows,
|
|
73
|
-
prompt,
|
|
74
|
-
config,
|
|
75
|
-
ctx_config,
|
|
76
|
-
user_token,
|
|
77
|
-
tb_client,
|
|
78
|
-
format_,
|
|
79
|
-
folder,
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
|
|
83
|
-
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
|
|
84
35
|
if env == "cloud":
|
|
85
|
-
|
|
36
|
+
prompt += "Append the fixture data to the datasource in Tinybird Cloud."
|
|
86
37
|
|
|
87
|
-
|
|
38
|
+
run_agent(ctx_config, project, True, prompt=prompt, feature="tb_mock")
|
|
88
39
|
|
|
89
40
|
except Exception as e:
|
|
90
41
|
raise CLIMockException(FeedbackManager.error(message=str(e)))
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
from typing import Any, Dict, List
|
|
2
|
-
|
|
3
|
-
from tinybird.prompts import mock_prompt
|
|
4
1
|
from tinybird.tb.client import TinyB
|
|
5
2
|
from tinybird.tb.modules.common import push_data
|
|
6
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
7
|
-
from tinybird.tb.modules.datafile.fixture import persist_fixture_sql
|
|
8
|
-
from tinybird.tb.modules.llm import LLM
|
|
9
|
-
from tinybird.tb.modules.llm_utils import extract_xml
|
|
10
3
|
|
|
11
4
|
|
|
12
5
|
def append_mock_data(
|
|
@@ -22,50 +15,3 @@ def append_mock_data(
|
|
|
22
15
|
concurrency=1,
|
|
23
16
|
silent=True,
|
|
24
17
|
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def create_mock_data(
|
|
28
|
-
datasource_name: str,
|
|
29
|
-
datasource_content: str,
|
|
30
|
-
rows: int,
|
|
31
|
-
prompt: str,
|
|
32
|
-
config: CLIConfig,
|
|
33
|
-
ctx_config: Dict[str, Any],
|
|
34
|
-
user_token: str,
|
|
35
|
-
tb_client: TinyB,
|
|
36
|
-
format_: str,
|
|
37
|
-
folder: str,
|
|
38
|
-
) -> List[Dict[str, Any]]:
|
|
39
|
-
user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
|
|
40
|
-
llm = LLM(user_token=user_token, host=user_client.host)
|
|
41
|
-
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
42
|
-
sql = ""
|
|
43
|
-
attempts = 0
|
|
44
|
-
data = []
|
|
45
|
-
error = ""
|
|
46
|
-
sql_path = None
|
|
47
|
-
while True:
|
|
48
|
-
try:
|
|
49
|
-
response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt, feature="tb_mock")
|
|
50
|
-
sql = extract_xml(response, "sql")
|
|
51
|
-
sql_path = persist_fixture_sql(datasource_name, sql, folder)
|
|
52
|
-
sql_format = "JSON" if format_ == "ndjson" else "CSV"
|
|
53
|
-
result = tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
|
|
54
|
-
if sql_format == "JSON":
|
|
55
|
-
data = result.get("data", [])[:rows]
|
|
56
|
-
error_response = result.get("error", None)
|
|
57
|
-
if error_response:
|
|
58
|
-
raise Exception(error_response)
|
|
59
|
-
else:
|
|
60
|
-
data = result
|
|
61
|
-
break
|
|
62
|
-
except Exception as e:
|
|
63
|
-
error = str(e)
|
|
64
|
-
attempts += 1
|
|
65
|
-
if attempts > 5:
|
|
66
|
-
raise Exception(
|
|
67
|
-
f"Failed to generate a valid solution. Check {str(sql_path or '.sql path')} and try again."
|
|
68
|
-
)
|
|
69
|
-
else:
|
|
70
|
-
continue
|
|
71
|
-
return data
|
tinybird/tb/modules/shell.py
CHANGED
|
@@ -294,7 +294,7 @@ class Shell:
|
|
|
294
294
|
arg = argline.strip().lower()
|
|
295
295
|
if not arg:
|
|
296
296
|
return
|
|
297
|
-
if arg.startswith("with"
|
|
297
|
+
if arg.startswith(("with", "select")):
|
|
298
298
|
self.run_sql(argline)
|
|
299
299
|
elif len(arg.split()) == 1 and arg in self.project.pipes + self.project.datasources:
|
|
300
300
|
self.run_sql(f"select * from {argline}")
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -8,9 +8,10 @@ from typing import Any, Tuple
|
|
|
8
8
|
import click
|
|
9
9
|
|
|
10
10
|
from tinybird.tb.client import TinyB
|
|
11
|
+
from tinybird.tb.modules.agent import run_agent
|
|
11
12
|
from tinybird.tb.modules.cli import cli
|
|
12
13
|
from tinybird.tb.modules.project import Project
|
|
13
|
-
from tinybird.tb.modules.test_common import
|
|
14
|
+
from tinybird.tb.modules.test_common import run_tests, update_test
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
@cli.group()
|
|
@@ -24,18 +25,18 @@ def test(ctx: click.Context) -> None:
|
|
|
24
25
|
help="Create a test for an existing pipe",
|
|
25
26
|
)
|
|
26
27
|
@click.argument("name_or_filename", type=str)
|
|
27
|
-
@click.option(
|
|
28
|
-
"--prompt", type=str, default="Create a test for the selected pipe", help="Prompt to be used to create the test"
|
|
29
|
-
)
|
|
28
|
+
@click.option("--prompt", type=str, default="", help="Prompt to be used to create the test")
|
|
30
29
|
@click.pass_context
|
|
31
30
|
def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
32
31
|
"""
|
|
33
32
|
Create a test for an existing pipe
|
|
34
33
|
"""
|
|
35
34
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
35
|
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
38
|
-
|
|
36
|
+
prompt = (
|
|
37
|
+
f"""Create tests for the following pipe: {name_or_filename}. Extra context: {prompt or "No extra context."}"""
|
|
38
|
+
)
|
|
39
|
+
run_agent(config, project, True, prompt=prompt, feature="tb_test_create")
|
|
39
40
|
|
|
40
41
|
|
|
41
42
|
@test.command(
|
|
@@ -14,14 +14,11 @@ import click
|
|
|
14
14
|
import yaml
|
|
15
15
|
from requests import Response
|
|
16
16
|
|
|
17
|
-
from tinybird.prompts import test_create_prompt
|
|
18
17
|
from tinybird.tb.client import TinyB
|
|
19
18
|
from tinybird.tb.modules.build_common import process as build_project
|
|
20
19
|
from tinybird.tb.modules.common import sys_exit
|
|
21
20
|
from tinybird.tb.modules.exceptions import CLITestException
|
|
22
21
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
23
|
-
from tinybird.tb.modules.llm import LLM
|
|
24
|
-
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
25
22
|
from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
|
|
26
23
|
from tinybird.tb.modules.project import Project
|
|
27
24
|
|
|
@@ -53,92 +50,6 @@ def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Opti
|
|
|
53
50
|
return path
|
|
54
51
|
|
|
55
52
|
|
|
56
|
-
def create_test(
|
|
57
|
-
name_or_filename: str, prompt: str, project: Project, client: TinyB, config: dict[str, Any], preview: bool = False
|
|
58
|
-
) -> list[dict[str, Any]]:
|
|
59
|
-
"""
|
|
60
|
-
Create a test for an existing pipe
|
|
61
|
-
"""
|
|
62
|
-
tests: List[Dict[str, Any]] = []
|
|
63
|
-
|
|
64
|
-
try:
|
|
65
|
-
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
66
|
-
build_error = build_project(
|
|
67
|
-
project=project,
|
|
68
|
-
tb_client=client,
|
|
69
|
-
watch=False,
|
|
70
|
-
silent=True,
|
|
71
|
-
exit_on_error=False,
|
|
72
|
-
config=config,
|
|
73
|
-
load_fixtures=True,
|
|
74
|
-
)
|
|
75
|
-
if build_error:
|
|
76
|
-
raise Exception(build_error)
|
|
77
|
-
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
78
|
-
folder = project.folder
|
|
79
|
-
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
80
|
-
pipe_name = pipe_path.stem
|
|
81
|
-
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
82
|
-
pipe_content = pipe_path.read_text()
|
|
83
|
-
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
84
|
-
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
85
|
-
|
|
86
|
-
system_prompt = test_create_prompt.format(
|
|
87
|
-
name=pipe_name,
|
|
88
|
-
content=pipe_content,
|
|
89
|
-
parameters=parameters or "No parameters",
|
|
90
|
-
)
|
|
91
|
-
user_token = config.get("user_token")
|
|
92
|
-
if not user_token:
|
|
93
|
-
raise Exception("No user token found")
|
|
94
|
-
|
|
95
|
-
llm = LLM(user_token=user_token, host=config.get("host") or "")
|
|
96
|
-
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
97
|
-
response_xml = extract_xml(response_llm, "response")
|
|
98
|
-
tests_content = parse_xml(response_xml, "test")
|
|
99
|
-
|
|
100
|
-
for test_content in tests_content:
|
|
101
|
-
test: Dict[str, Any] = {}
|
|
102
|
-
test["name"] = extract_xml(test_content, "name")
|
|
103
|
-
test["description"] = extract_xml(test_content, "description")
|
|
104
|
-
parameters_api = extract_xml(test_content, "parameters")
|
|
105
|
-
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
106
|
-
test["expected_result"] = ""
|
|
107
|
-
|
|
108
|
-
response = None
|
|
109
|
-
try:
|
|
110
|
-
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
111
|
-
except Exception:
|
|
112
|
-
pass
|
|
113
|
-
|
|
114
|
-
if response:
|
|
115
|
-
if response.status_code >= 400:
|
|
116
|
-
test["expected_http_status"] = response.status_code
|
|
117
|
-
test["expected_result"] = response.json()["error"]
|
|
118
|
-
else:
|
|
119
|
-
test.pop("expected_http_status", None)
|
|
120
|
-
test["expected_result"] = response.text or ""
|
|
121
|
-
|
|
122
|
-
tests.append(test)
|
|
123
|
-
|
|
124
|
-
if not preview:
|
|
125
|
-
if len(tests) > 0:
|
|
126
|
-
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
127
|
-
for test in tests:
|
|
128
|
-
test_name = test["name"]
|
|
129
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
130
|
-
else:
|
|
131
|
-
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
132
|
-
|
|
133
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
134
|
-
except Exception as e:
|
|
135
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
136
|
-
finally:
|
|
137
|
-
cleanup_test_workspace(client, project.folder)
|
|
138
|
-
|
|
139
|
-
return tests
|
|
140
|
-
|
|
141
|
-
|
|
142
53
|
def parse_tests(tests_content: str) -> List[Dict[str, Any]]:
|
|
143
54
|
return yaml.safe_load(tests_content)
|
|
144
55
|
|
|
@@ -1042,7 +1042,7 @@ def get_format_from_filename_or_url(filename_or_url: str) -> str:
|
|
|
1042
1042
|
'csv'
|
|
1043
1043
|
"""
|
|
1044
1044
|
filename_or_url = filename_or_url.lower()
|
|
1045
|
-
if filename_or_url.endswith("json"
|
|
1045
|
+
if filename_or_url.endswith(("json", "ndjson")):
|
|
1046
1046
|
return "ndjson"
|
|
1047
1047
|
if filename_or_url.endswith("parquet"):
|
|
1048
1048
|
return "parquet"
|
|
@@ -1050,7 +1050,7 @@ def get_format_from_filename_or_url(filename_or_url: str) -> str:
|
|
|
1050
1050
|
return "csv"
|
|
1051
1051
|
try:
|
|
1052
1052
|
parsed = urlparse(filename_or_url)
|
|
1053
|
-
if parsed.path.endswith("json"
|
|
1053
|
+
if parsed.path.endswith(("json", "ndjson")):
|
|
1054
1054
|
return "ndjson"
|
|
1055
1055
|
if parsed.path.endswith("parquet"):
|
|
1056
1056
|
return "parquet"
|