tinybird 0.0.1.dev43__py3-none-any.whl → 0.0.1.dev44__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/prompts.py +30 -3
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -1
- tinybird/tb/modules/build.py +21 -16
- tinybird/tb/modules/cli.py +2 -57
- tinybird/tb/modules/common.py +2 -1
- tinybird/tb/modules/copy.py +1 -1
- tinybird/tb/modules/create.py +105 -46
- tinybird/tb/modules/datafile/build.py +12 -221
- tinybird/tb/modules/datasource.py +1 -1
- tinybird/tb/modules/deployment.py +1 -1
- tinybird/tb/modules/endpoint.py +89 -2
- tinybird/tb/modules/materialization.py +146 -0
- tinybird/tb/modules/mock.py +56 -16
- tinybird/tb/modules/pipe.py +2 -321
- tinybird/tb/modules/project.py +10 -4
- tinybird/tb/modules/test.py +72 -37
- tinybird/tb/modules/update.py +1 -1
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev44.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev44.dist-info}/RECORD +23 -23
- tinybird/tb/modules/build_client.py +0 -199
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev44.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev44.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev43.dist-info → tinybird-0.0.1.dev44.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/mock.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
from pathlib import Path
|
|
4
|
+
from typing import Optional
|
|
4
5
|
|
|
5
6
|
import click
|
|
6
7
|
|
|
@@ -13,6 +14,7 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
|
13
14
|
from tinybird.tb.modules.llm import LLM
|
|
14
15
|
from tinybird.tb.modules.llm_utils import extract_xml
|
|
15
16
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
17
|
+
from tinybird.tb.modules.project import Project
|
|
16
18
|
|
|
17
19
|
|
|
18
20
|
@cli.command()
|
|
@@ -21,24 +23,27 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
|
21
23
|
@click.option(
|
|
22
24
|
"--prompt",
|
|
23
25
|
type=str,
|
|
24
|
-
default="
|
|
26
|
+
default="",
|
|
25
27
|
help="Extra context to use for data generation",
|
|
26
28
|
)
|
|
27
|
-
@click.option("--
|
|
29
|
+
@click.option("--skip", is_flag=True, default=False, help="Skip following up on the generated data")
|
|
30
|
+
@click.pass_context
|
|
28
31
|
@coro
|
|
29
|
-
async def mock(datasource: str, rows: int, prompt: str,
|
|
30
|
-
"""
|
|
32
|
+
async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, skip: bool) -> None:
|
|
33
|
+
"""Generate sample data for a datasource.
|
|
31
34
|
|
|
32
35
|
Args:
|
|
33
36
|
datasource: Path to the datasource file to load sample data into
|
|
34
37
|
rows: Number of events to send
|
|
35
38
|
prompt: Extra context to use for data generation
|
|
36
|
-
|
|
39
|
+
skip: Skip following up on the generated data
|
|
37
40
|
"""
|
|
38
41
|
|
|
39
42
|
try:
|
|
43
|
+
project: Project = ctx.ensure_object(dict)["project"]
|
|
40
44
|
datasource_path = Path(datasource)
|
|
41
45
|
datasource_name = datasource
|
|
46
|
+
folder = project.folder
|
|
42
47
|
click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
|
|
43
48
|
if datasource_path.suffix == ".datasource":
|
|
44
49
|
datasource_name = datasource_path.stem
|
|
@@ -46,6 +51,9 @@ async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
|
|
|
46
51
|
datasource_path = Path("datasources", f"{datasource}.datasource")
|
|
47
52
|
datasource_path = Path(folder) / datasource_path
|
|
48
53
|
|
|
54
|
+
if not datasource_path.exists():
|
|
55
|
+
raise CLIException(f"Datasource '{datasource_path.stem}' not found")
|
|
56
|
+
|
|
49
57
|
prompt_path = Path(folder) / "fixtures" / f"{datasource_name}.prompt"
|
|
50
58
|
if not prompt or prompt == "Use the datasource schema to generate sample data":
|
|
51
59
|
# load the prompt from the fixture.prompt file if it exists
|
|
@@ -68,17 +76,49 @@ async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
|
|
|
68
76
|
click.echo(FeedbackManager.error(message="This action requires authentication. Run 'tb login' first."))
|
|
69
77
|
return
|
|
70
78
|
llm = LLM(user_token=user_token, host=user_client.host)
|
|
71
|
-
tb_client = await get_tinybird_local_client(
|
|
79
|
+
tb_client = await get_tinybird_local_client(folder)
|
|
72
80
|
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
iterations = 0
|
|
82
|
+
history = ""
|
|
83
|
+
fixture_path: Optional[Path] = None
|
|
84
|
+
sql = ""
|
|
85
|
+
while iterations < 10:
|
|
86
|
+
feedback = ""
|
|
87
|
+
if iterations > 0:
|
|
88
|
+
feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
|
|
89
|
+
if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
|
|
90
|
+
break
|
|
91
|
+
else:
|
|
92
|
+
if iterations > 0:
|
|
93
|
+
if fixture_path:
|
|
94
|
+
fixture_path.unlink()
|
|
95
|
+
fixture_path = None
|
|
96
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Creating fixture for {datasource_name}..."))
|
|
97
|
+
|
|
98
|
+
response = llm.ask(system_prompt=mock_prompt(rows, feedback, history), prompt=prompt)
|
|
99
|
+
sql = extract_xml(response, "sql")
|
|
100
|
+
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
101
|
+
data = result.get("data", [])[:rows]
|
|
102
|
+
fixture_name = build_fixture_name(str(datasource_path), datasource_name, datasource_content)
|
|
103
|
+
fixture_path = persist_fixture(fixture_name, data, folder)
|
|
104
|
+
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{fixture_name}.ndjson created"))
|
|
105
|
+
|
|
106
|
+
if os.environ.get("TB_DEBUG", "") != "":
|
|
107
|
+
logging.debug(sql)
|
|
108
|
+
|
|
109
|
+
history = (
|
|
110
|
+
history
|
|
111
|
+
+ f"""
|
|
112
|
+
<result_iteration_{iterations}>
|
|
113
|
+
{response}
|
|
114
|
+
</result_iteration_{iterations}>
|
|
115
|
+
"""
|
|
116
|
+
)
|
|
117
|
+
if skip:
|
|
118
|
+
break
|
|
119
|
+
iterations += 1
|
|
120
|
+
|
|
121
|
+
click.echo(FeedbackManager.success(message=f"✓ Sample data for {datasource_name} created with {rows} rows"))
|
|
82
122
|
|
|
83
123
|
except Exception as e:
|
|
84
|
-
|
|
124
|
+
click.echo(FeedbackManager.error_exception(error=f"Error: {e}"))
|
tinybird/tb/modules/pipe.py
CHANGED
|
@@ -5,21 +5,17 @@
|
|
|
5
5
|
|
|
6
6
|
import json
|
|
7
7
|
import re
|
|
8
|
-
from typing import Dict, List, Optional, Tuple
|
|
9
8
|
|
|
10
9
|
import click
|
|
11
|
-
import humanfriendly
|
|
12
10
|
from click import Context
|
|
13
11
|
|
|
14
|
-
from tinybird.client import
|
|
12
|
+
from tinybird.client import TinyB
|
|
15
13
|
from tinybird.tb.modules.cli import cli
|
|
16
14
|
from tinybird.tb.modules.common import (
|
|
17
15
|
coro,
|
|
18
|
-
create_tb_client,
|
|
19
16
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
20
|
-
wait_job,
|
|
21
17
|
)
|
|
22
|
-
from tinybird.tb.modules.datafile.common import
|
|
18
|
+
from tinybird.tb.modules.datafile.common import get_name_version
|
|
23
19
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
24
20
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
25
21
|
|
|
@@ -30,105 +26,6 @@ def pipe(ctx):
|
|
|
30
26
|
"""Pipes commands"""
|
|
31
27
|
|
|
32
28
|
|
|
33
|
-
@pipe.group(name="copy")
|
|
34
|
-
@click.pass_context
|
|
35
|
-
def pipe_copy(ctx: Context) -> None:
|
|
36
|
-
"""Copy Pipe commands"""
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@pipe.group(name="sink")
|
|
40
|
-
@click.pass_context
|
|
41
|
-
def pipe_sink(ctx: Context) -> None:
|
|
42
|
-
"""Sink Pipe commands"""
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
@pipe.command(name="stats")
|
|
46
|
-
@click.argument("pipes", nargs=-1)
|
|
47
|
-
@click.option(
|
|
48
|
-
"--format",
|
|
49
|
-
"format_",
|
|
50
|
-
type=click.Choice(["json"], case_sensitive=False),
|
|
51
|
-
default=None,
|
|
52
|
-
help="Force a type of the output. To parse the output, keep in mind to use `tb --no-version-warning pipe stats` option.",
|
|
53
|
-
)
|
|
54
|
-
@click.pass_context
|
|
55
|
-
@coro
|
|
56
|
-
async def pipe_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: str):
|
|
57
|
-
"""
|
|
58
|
-
Print pipe stats for the last 7 days
|
|
59
|
-
"""
|
|
60
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
61
|
-
all_pipes = await client.pipes()
|
|
62
|
-
pipes_to_get_stats = []
|
|
63
|
-
pipes_ids: Dict = {}
|
|
64
|
-
|
|
65
|
-
if pipes:
|
|
66
|
-
# We filter by the pipes we want to look for
|
|
67
|
-
all_pipes = [pipe for pipe in all_pipes if pipe["name"] in pipes]
|
|
68
|
-
|
|
69
|
-
for pipe in all_pipes:
|
|
70
|
-
name_version = get_name_version(pipe["name"])
|
|
71
|
-
if name_version["name"] in pipe["name"]:
|
|
72
|
-
pipes_to_get_stats.append(f"'{pipe['id']}'")
|
|
73
|
-
pipes_ids[pipe["id"]] = name_version
|
|
74
|
-
|
|
75
|
-
if not pipes_to_get_stats:
|
|
76
|
-
if format_ == "json":
|
|
77
|
-
click.echo(json.dumps({"pipes": []}, indent=2))
|
|
78
|
-
else:
|
|
79
|
-
click.echo(FeedbackManager.info_no_pipes_stats())
|
|
80
|
-
return
|
|
81
|
-
|
|
82
|
-
sql = f"""
|
|
83
|
-
SELECT
|
|
84
|
-
pipe_id id,
|
|
85
|
-
sumIf(view_count, date > now() - interval 7 day) requests,
|
|
86
|
-
sumIf(error_count, date > now() - interval 7 day) errors,
|
|
87
|
-
avgMergeIf(avg_duration_state, date > now() - interval 7 day) latency
|
|
88
|
-
FROM tinybird.pipe_stats
|
|
89
|
-
WHERE pipe_id in ({','.join(pipes_to_get_stats)})
|
|
90
|
-
GROUP BY pipe_id
|
|
91
|
-
ORDER BY requests DESC
|
|
92
|
-
FORMAT JSON
|
|
93
|
-
"""
|
|
94
|
-
|
|
95
|
-
res = await client.query(sql)
|
|
96
|
-
|
|
97
|
-
if res and "error" in res:
|
|
98
|
-
raise CLIPipeException(FeedbackManager.error_exception(error=str(res["error"])))
|
|
99
|
-
|
|
100
|
-
columns = ["name", "request count", "error count", "avg latency"]
|
|
101
|
-
table_human_readable: List[Tuple] = []
|
|
102
|
-
table_machine_readable: List[Dict] = []
|
|
103
|
-
if res and "data" in res:
|
|
104
|
-
for x in res["data"]:
|
|
105
|
-
tk = pipes_ids[x["id"]]
|
|
106
|
-
table_human_readable.append(
|
|
107
|
-
(
|
|
108
|
-
tk["name"],
|
|
109
|
-
x["requests"],
|
|
110
|
-
x["errors"],
|
|
111
|
-
x["latency"],
|
|
112
|
-
)
|
|
113
|
-
)
|
|
114
|
-
table_machine_readable.append(
|
|
115
|
-
{
|
|
116
|
-
"name": tk["name"],
|
|
117
|
-
"requests": x["requests"],
|
|
118
|
-
"errors": x["errors"],
|
|
119
|
-
"latency": x["latency"],
|
|
120
|
-
}
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
table_human_readable.sort(key=lambda x: (x[1], x[0]))
|
|
124
|
-
table_machine_readable.sort(key=lambda x: x["name"])
|
|
125
|
-
|
|
126
|
-
if format_ == "json":
|
|
127
|
-
click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
|
|
128
|
-
else:
|
|
129
|
-
echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
|
|
130
|
-
|
|
131
|
-
|
|
132
29
|
@pipe.command(name="ls")
|
|
133
30
|
@click.option("--match", default=None, help="Retrieve any resourcing matching the pattern. eg --match _test")
|
|
134
31
|
@click.option(
|
|
@@ -172,219 +69,3 @@ async def pipe_ls(ctx: Context, match: str, format_: str):
|
|
|
172
69
|
click.echo(json.dumps({"pipes": table_machine_readable}, indent=2))
|
|
173
70
|
else:
|
|
174
71
|
raise CLIPipeException(FeedbackManager.error_pipe_ls_type())
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
@pipe.command(name="populate")
|
|
178
|
-
@click.argument("pipe_name")
|
|
179
|
-
@click.option("--node", type=str, help="Name of the materialized node.", default=None, required=False)
|
|
180
|
-
@click.option(
|
|
181
|
-
"--sql-condition",
|
|
182
|
-
type=str,
|
|
183
|
-
default=None,
|
|
184
|
-
help="Populate with a SQL condition to be applied to the trigger Data Source of the Materialized View. For instance, `--sql-condition='date == toYYYYMM(now())'` it'll populate taking all the rows from the trigger Data Source which `date` is the current month. Use it together with --populate. --sql-condition is not taken into account if the --subset param is present. Including in the ``sql_condition`` any column present in the Data Source ``engine_sorting_key`` will make the populate job process less data.",
|
|
185
|
-
)
|
|
186
|
-
@click.option(
|
|
187
|
-
"--truncate", is_flag=True, default=False, help="Truncates the materialized Data Source before populating it."
|
|
188
|
-
)
|
|
189
|
-
@click.option(
|
|
190
|
-
"--unlink-on-populate-error",
|
|
191
|
-
is_flag=True,
|
|
192
|
-
default=False,
|
|
193
|
-
help="If the populate job fails the Materialized View is unlinked and new data won't be ingested in the Materialized View. First time a populate job fails, the Materialized View is always unlinked.",
|
|
194
|
-
)
|
|
195
|
-
@click.option(
|
|
196
|
-
"--wait",
|
|
197
|
-
is_flag=True,
|
|
198
|
-
default=False,
|
|
199
|
-
help="Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
|
|
200
|
-
)
|
|
201
|
-
@click.pass_context
|
|
202
|
-
@coro
|
|
203
|
-
async def pipe_populate(
|
|
204
|
-
ctx: click.Context,
|
|
205
|
-
pipe_name: str,
|
|
206
|
-
node: str,
|
|
207
|
-
sql_condition: str,
|
|
208
|
-
truncate: bool,
|
|
209
|
-
unlink_on_populate_error: bool,
|
|
210
|
-
wait: bool,
|
|
211
|
-
):
|
|
212
|
-
"""Populate the result of a Materialized Node into the target Materialized View"""
|
|
213
|
-
cl = create_tb_client(ctx)
|
|
214
|
-
|
|
215
|
-
pipe = await cl.pipe(pipe_name)
|
|
216
|
-
|
|
217
|
-
if pipe["type"] != PipeTypes.MATERIALIZED:
|
|
218
|
-
raise CLIPipeException(FeedbackManager.error_pipe_not_materialized(pipe=pipe_name))
|
|
219
|
-
|
|
220
|
-
if not node:
|
|
221
|
-
materialized_ids = [pipe_node["id"] for pipe_node in pipe["nodes"] if pipe_node.get("materialized") is not None]
|
|
222
|
-
|
|
223
|
-
if not materialized_ids:
|
|
224
|
-
raise CLIPipeException(FeedbackManager.error_populate_no_materialized_in_pipe(pipe=pipe_name))
|
|
225
|
-
|
|
226
|
-
elif len(materialized_ids) > 1:
|
|
227
|
-
raise CLIPipeException(FeedbackManager.error_populate_several_materialized_in_pipe(pipe=pipe_name))
|
|
228
|
-
|
|
229
|
-
node = materialized_ids[0]
|
|
230
|
-
|
|
231
|
-
response = await cl.populate_node(
|
|
232
|
-
pipe_name,
|
|
233
|
-
node,
|
|
234
|
-
populate_condition=sql_condition,
|
|
235
|
-
truncate=truncate,
|
|
236
|
-
unlink_on_populate_error=unlink_on_populate_error,
|
|
237
|
-
)
|
|
238
|
-
if "job" not in response:
|
|
239
|
-
raise CLIPipeException(response)
|
|
240
|
-
|
|
241
|
-
job_id = response["job"]["id"]
|
|
242
|
-
job_url = response["job"]["job_url"]
|
|
243
|
-
if sql_condition:
|
|
244
|
-
click.echo(FeedbackManager.info_populate_condition_job_url(url=job_url, populate_condition=sql_condition))
|
|
245
|
-
else:
|
|
246
|
-
click.echo(FeedbackManager.info_populate_job_url(url=job_url))
|
|
247
|
-
if wait:
|
|
248
|
-
await wait_job(cl, job_id, job_url, "Populating")
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
@pipe.command(name="token_read")
|
|
252
|
-
@click.argument("pipe_name")
|
|
253
|
-
@click.pass_context
|
|
254
|
-
@coro
|
|
255
|
-
async def pipe_token_read(ctx: click.Context, pipe_name: str):
|
|
256
|
-
"""Retrieve a token to read a pipe"""
|
|
257
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
258
|
-
|
|
259
|
-
try:
|
|
260
|
-
await client.pipe_file(pipe_name)
|
|
261
|
-
except DoesNotExistException:
|
|
262
|
-
raise CLIPipeException(FeedbackManager.error_pipe_does_not_exist(pipe=pipe_name))
|
|
263
|
-
|
|
264
|
-
tokens = await client.tokens()
|
|
265
|
-
token = None
|
|
266
|
-
|
|
267
|
-
for t in tokens:
|
|
268
|
-
for scope in t["scopes"]:
|
|
269
|
-
if scope["type"] == "PIPES:READ" and scope["resource"] == pipe_name:
|
|
270
|
-
token = t["token"]
|
|
271
|
-
if token:
|
|
272
|
-
click.echo(token)
|
|
273
|
-
else:
|
|
274
|
-
click.echo(FeedbackManager.warning_token_pipe(pipe=pipe_name))
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
@pipe.command(
|
|
278
|
-
name="data",
|
|
279
|
-
context_settings=dict(
|
|
280
|
-
allow_extra_args=True,
|
|
281
|
-
ignore_unknown_options=True,
|
|
282
|
-
),
|
|
283
|
-
)
|
|
284
|
-
@click.argument("pipe")
|
|
285
|
-
@click.option("--query", default=None, help="Run SQL over pipe results")
|
|
286
|
-
@click.option(
|
|
287
|
-
"--format", "format_", type=click.Choice(["json", "csv"], case_sensitive=False), help="Return format (CSV, JSON)"
|
|
288
|
-
)
|
|
289
|
-
@click.pass_context
|
|
290
|
-
@coro
|
|
291
|
-
async def print_pipe(ctx: Context, pipe: str, query: str, format_: str):
|
|
292
|
-
"""Print data returned by a pipe
|
|
293
|
-
|
|
294
|
-
Syntax: tb pipe data <pipe_name> --param_name value --param2_name value2 ...
|
|
295
|
-
"""
|
|
296
|
-
|
|
297
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
298
|
-
params = {ctx.args[i][2:]: ctx.args[i + 1] for i in range(0, len(ctx.args), 2)}
|
|
299
|
-
req_format = "json" if not format_ else format_.lower()
|
|
300
|
-
try:
|
|
301
|
-
res = await client.pipe_data(pipe, format=req_format, sql=query, params=params)
|
|
302
|
-
except AuthNoTokenException:
|
|
303
|
-
raise
|
|
304
|
-
except Exception as e:
|
|
305
|
-
raise CLIPipeException(FeedbackManager.error_exception(error=str(e)))
|
|
306
|
-
|
|
307
|
-
if not format_:
|
|
308
|
-
stats = res["statistics"]
|
|
309
|
-
seconds = stats["elapsed"]
|
|
310
|
-
rows_read = humanfriendly.format_number(stats["rows_read"])
|
|
311
|
-
bytes_read = humanfriendly.format_size(stats["bytes_read"])
|
|
312
|
-
|
|
313
|
-
click.echo(FeedbackManager.success_print_pipe(pipe=pipe))
|
|
314
|
-
click.echo(FeedbackManager.info_query_stats(seconds=seconds, rows=rows_read, bytes=bytes_read))
|
|
315
|
-
|
|
316
|
-
if not res["data"]:
|
|
317
|
-
click.echo(FeedbackManager.info_no_rows())
|
|
318
|
-
else:
|
|
319
|
-
echo_safe_humanfriendly_tables_format_smart_table(
|
|
320
|
-
data=[d.values() for d in res["data"]], column_names=res["data"][0].keys()
|
|
321
|
-
)
|
|
322
|
-
click.echo("\n")
|
|
323
|
-
elif req_format == "json":
|
|
324
|
-
click.echo(json.dumps(res))
|
|
325
|
-
else:
|
|
326
|
-
click.echo(res)
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
@pipe_sink.command(name="run", short_help="Run an on-demand sink job")
|
|
330
|
-
@click.argument("pipe_name_or_id")
|
|
331
|
-
@click.option("--wait", is_flag=True, default=False, help="Wait for the sink job to finish")
|
|
332
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
333
|
-
@click.option("--dry-run", is_flag=True, default=False, help="Run the command without executing the sink job")
|
|
334
|
-
@click.option(
|
|
335
|
-
"--param",
|
|
336
|
-
nargs=1,
|
|
337
|
-
type=str,
|
|
338
|
-
multiple=True,
|
|
339
|
-
default=None,
|
|
340
|
-
help="Key and value of the params you want the Sink pipe to be called with. For example: tb pipe sink run <my_sink_pipe> --param foo=bar",
|
|
341
|
-
)
|
|
342
|
-
@click.pass_context
|
|
343
|
-
@coro
|
|
344
|
-
async def pipe_sink_run(
|
|
345
|
-
ctx: click.Context, pipe_name_or_id: str, wait: bool, yes: bool, dry_run: bool, param: Optional[Tuple[str]]
|
|
346
|
-
):
|
|
347
|
-
"""Run an on-demand sink job"""
|
|
348
|
-
|
|
349
|
-
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
350
|
-
|
|
351
|
-
if dry_run or yes or click.confirm(FeedbackManager.warning_confirm_sink_job(pipe=pipe_name_or_id)):
|
|
352
|
-
click.echo(FeedbackManager.info_sink_job_running(pipe=pipe_name_or_id))
|
|
353
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
354
|
-
|
|
355
|
-
try:
|
|
356
|
-
pipe = await client.pipe(pipe_name_or_id)
|
|
357
|
-
connections = await client.get_connections()
|
|
358
|
-
|
|
359
|
-
if (pipe.get("type", None) != "sink") or (not pipe.get("sink_node", None)):
|
|
360
|
-
error_message = f"Pipe {pipe_name_or_id} is not published as a Sink pipe"
|
|
361
|
-
raise Exception(FeedbackManager.error_running_on_demand_sink_job(error=error_message))
|
|
362
|
-
|
|
363
|
-
current_sink = None
|
|
364
|
-
for connection in connections:
|
|
365
|
-
for sink in connection.get("sinks", []):
|
|
366
|
-
if sink.get("resource_id") == pipe["id"]:
|
|
367
|
-
current_sink = sink
|
|
368
|
-
break
|
|
369
|
-
|
|
370
|
-
if not current_sink:
|
|
371
|
-
click.echo(FeedbackManager.warning_sink_no_connection(pipe_name=pipe.get("name", "")))
|
|
372
|
-
|
|
373
|
-
if dry_run:
|
|
374
|
-
click.echo(FeedbackManager.info_dry_sink_run())
|
|
375
|
-
return
|
|
376
|
-
|
|
377
|
-
bucket_path = (current_sink or {}).get("settings", {}).get("bucket_path", "")
|
|
378
|
-
response = await client.pipe_run_sink(pipe_name_or_id, params)
|
|
379
|
-
job_id = response["job"]["id"]
|
|
380
|
-
job_url = response["job"]["job_url"]
|
|
381
|
-
click.echo(FeedbackManager.success_sink_job_created(bucket_path=bucket_path, job_url=job_url))
|
|
382
|
-
|
|
383
|
-
if wait:
|
|
384
|
-
await wait_job(client, job_id, job_url, "** Sinking data")
|
|
385
|
-
click.echo(FeedbackManager.success_sink_job_finished(bucket_path=bucket_path))
|
|
386
|
-
|
|
387
|
-
except AuthNoTokenException:
|
|
388
|
-
raise
|
|
389
|
-
except Exception as e:
|
|
390
|
-
raise CLIPipeException(FeedbackManager.error_creating_sink_job(error=str(e)))
|
tinybird/tb/modules/project.py
CHANGED
|
@@ -46,11 +46,17 @@ class Project:
|
|
|
46
46
|
def pipes(self) -> List[str]:
|
|
47
47
|
return [Path(f).stem for f in glob.glob(f"{self.path}/**/*.pipe", recursive=True)]
|
|
48
48
|
|
|
49
|
-
def get_pipe_datafile(self, filename: str) -> Datafile:
|
|
50
|
-
|
|
49
|
+
def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
|
|
50
|
+
try:
|
|
51
|
+
return parse_pipe(filename)
|
|
52
|
+
except Exception:
|
|
53
|
+
return None
|
|
51
54
|
|
|
52
|
-
def get_datasource_datafile(self, filename: str) -> Datafile:
|
|
53
|
-
|
|
55
|
+
def get_datasource_datafile(self, filename: str) -> Optional[Datafile]:
|
|
56
|
+
try:
|
|
57
|
+
return parse_datasource(filename)
|
|
58
|
+
except Exception:
|
|
59
|
+
return None
|
|
54
60
|
|
|
55
61
|
def get_datafile(self, filename: str) -> Optional[Datafile]:
|
|
56
62
|
if filename.endswith(".pipe"):
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -37,7 +37,7 @@ def repr_str(dumper, data):
|
|
|
37
37
|
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w"):
|
|
40
|
+
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w") -> Path:
|
|
41
41
|
base = Path("tests")
|
|
42
42
|
if folder:
|
|
43
43
|
base = Path(folder) / base
|
|
@@ -50,6 +50,7 @@ def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Opti
|
|
|
50
50
|
path = base / f"{pipe_name}.yaml"
|
|
51
51
|
with open(path, mode) as f:
|
|
52
52
|
f.write(formatted_yaml)
|
|
53
|
+
return path
|
|
53
54
|
|
|
54
55
|
|
|
55
56
|
@cli.group()
|
|
@@ -66,9 +67,14 @@ def test(ctx: click.Context) -> None:
|
|
|
66
67
|
@click.option(
|
|
67
68
|
"--prompt", type=str, default="Create a test for the selected pipe", help="Prompt to be used to create the test"
|
|
68
69
|
)
|
|
70
|
+
@click.option(
|
|
71
|
+
"--skip",
|
|
72
|
+
is_flag=True,
|
|
73
|
+
help="Skip the test creation process and only generate the test file",
|
|
74
|
+
)
|
|
69
75
|
@click.pass_context
|
|
70
76
|
@coro
|
|
71
|
-
async def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
77
|
+
async def test_create(ctx: click.Context, name_or_filename: str, prompt: str, skip: bool) -> None:
|
|
72
78
|
"""
|
|
73
79
|
Create a test for an existing pipe
|
|
74
80
|
"""
|
|
@@ -111,43 +117,72 @@ async def test_create(ctx: click.Context, name_or_filename: str, prompt: str) ->
|
|
|
111
117
|
raise CLIException(FeedbackManager.error(message="No user token found"))
|
|
112
118
|
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
113
119
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
tests: List[Dict[str, Any]] = []
|
|
119
|
-
for test_content in tests_content:
|
|
120
|
-
test: Dict[str, Any] = {}
|
|
121
|
-
test["name"] = extract_xml(test_content, "name")
|
|
122
|
-
test["description"] = extract_xml(test_content, "description")
|
|
123
|
-
parameters_api = extract_xml(test_content, "parameters")
|
|
124
|
-
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
125
|
-
test["expected_result"] = ""
|
|
126
|
-
|
|
127
|
-
response = None
|
|
128
|
-
try:
|
|
129
|
-
response = await get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
130
|
-
except Exception:
|
|
131
|
-
pass
|
|
120
|
+
iterations = 0
|
|
121
|
+
history = ""
|
|
122
|
+
test_path: Optional[Path] = None
|
|
132
123
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
124
|
+
while iterations < 10:
|
|
125
|
+
feedback = ""
|
|
126
|
+
if iterations > 0:
|
|
127
|
+
feedback = click.prompt("\nFollow-up instructions or continue", default="continue")
|
|
128
|
+
if iterations > 0 and (not feedback or feedback in ("continue", "ok", "exit", "quit", "q")):
|
|
129
|
+
break
|
|
130
|
+
else:
|
|
131
|
+
if iterations > 0 and test_path:
|
|
132
|
+
test_path.unlink()
|
|
133
|
+
test_path = None
|
|
134
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Creating test for {pipe_name} endpoint..."))
|
|
135
|
+
|
|
136
|
+
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt)
|
|
137
|
+
response_xml = extract_xml(response_llm, "response")
|
|
138
|
+
tests_content = parse_xml(response_xml, "test")
|
|
139
|
+
|
|
140
|
+
tests: List[Dict[str, Any]] = []
|
|
141
|
+
|
|
142
|
+
for test_content in tests_content:
|
|
143
|
+
test: Dict[str, Any] = {}
|
|
144
|
+
test["name"] = extract_xml(test_content, "name")
|
|
145
|
+
test["description"] = extract_xml(test_content, "description")
|
|
146
|
+
parameters_api = extract_xml(test_content, "parameters")
|
|
147
|
+
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
148
|
+
test["expected_result"] = ""
|
|
149
|
+
|
|
150
|
+
response = None
|
|
151
|
+
try:
|
|
152
|
+
response = await get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
153
|
+
except Exception:
|
|
154
|
+
pass
|
|
155
|
+
|
|
156
|
+
if response:
|
|
157
|
+
if response.status_code >= 400:
|
|
158
|
+
test["expected_http_status"] = response.status_code
|
|
159
|
+
test["expected_result"] = response.json()["error"]
|
|
160
|
+
else:
|
|
161
|
+
if "expected_http_status" in test:
|
|
162
|
+
del test["expected_http_status"]
|
|
163
|
+
test["expected_result"] = response.text or ""
|
|
164
|
+
|
|
165
|
+
tests.append(test)
|
|
166
|
+
|
|
167
|
+
if len(tests) > 0:
|
|
168
|
+
test_path = generate_test_file(pipe_name, tests, folder, mode="a")
|
|
169
|
+
for test in tests:
|
|
170
|
+
test_name = test["name"]
|
|
171
|
+
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
172
|
+
|
|
173
|
+
history = (
|
|
174
|
+
history
|
|
175
|
+
+ f"""
|
|
176
|
+
<result_iteration_{iterations}>
|
|
177
|
+
{response_xml}
|
|
178
|
+
</result_iteration_{iterations}>
|
|
179
|
+
"""
|
|
180
|
+
)
|
|
181
|
+
if skip:
|
|
182
|
+
break
|
|
183
|
+
iterations += 1
|
|
137
184
|
else:
|
|
138
|
-
|
|
139
|
-
del test["expected_http_status"]
|
|
140
|
-
test["expected_result"] = response.text or ""
|
|
141
|
-
|
|
142
|
-
tests.append(test)
|
|
143
|
-
|
|
144
|
-
if len(tests) > 0:
|
|
145
|
-
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
146
|
-
for test in tests:
|
|
147
|
-
test_name = test["name"]
|
|
148
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
149
|
-
else:
|
|
150
|
-
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
185
|
+
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
151
186
|
|
|
152
187
|
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
153
188
|
except Exception as e:
|
tinybird/tb/modules/update.py
CHANGED