tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/ch_utils/constants.py +5 -0
- tinybird/connectors.py +1 -7
- tinybird/context.py +3 -3
- tinybird/datafile/common.py +10 -8
- tinybird/datafile/parse_pipe.py +2 -2
- tinybird/feedback_manager.py +3 -0
- tinybird/prompts.py +1 -0
- tinybird/service_datasources.py +223 -0
- tinybird/sql_template.py +26 -11
- tinybird/sql_template_fmt.py +14 -4
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -0
- tinybird/tb/client.py +104 -26
- tinybird/tb/config.py +24 -0
- tinybird/tb/modules/agent/agent.py +103 -67
- tinybird/tb/modules/agent/banner.py +15 -15
- tinybird/tb/modules/agent/explore_agent.py +5 -0
- tinybird/tb/modules/agent/mock_agent.py +5 -1
- tinybird/tb/modules/agent/models.py +6 -2
- tinybird/tb/modules/agent/prompts.py +49 -2
- tinybird/tb/modules/agent/tools/deploy.py +1 -1
- tinybird/tb/modules/agent/tools/execute_query.py +15 -18
- tinybird/tb/modules/agent/tools/request_endpoint.py +1 -1
- tinybird/tb/modules/agent/tools/run_command.py +9 -0
- tinybird/tb/modules/agent/utils.py +38 -48
- tinybird/tb/modules/branch.py +150 -0
- tinybird/tb/modules/build.py +58 -13
- tinybird/tb/modules/build_common.py +209 -25
- tinybird/tb/modules/cli.py +129 -16
- tinybird/tb/modules/common.py +172 -146
- tinybird/tb/modules/connection.py +125 -194
- tinybird/tb/modules/connection_kafka.py +382 -0
- tinybird/tb/modules/copy.py +3 -1
- tinybird/tb/modules/create.py +83 -150
- tinybird/tb/modules/datafile/build.py +27 -38
- tinybird/tb/modules/datafile/build_datasource.py +21 -25
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +46 -7
- tinybird/tb/modules/datafile/playground.py +59 -68
- tinybird/tb/modules/datafile/pull.py +2 -3
- tinybird/tb/modules/datasource.py +477 -308
- tinybird/tb/modules/deployment.py +2 -0
- tinybird/tb/modules/deployment_common.py +84 -44
- tinybird/tb/modules/deprecations.py +4 -4
- tinybird/tb/modules/dev_server.py +33 -12
- tinybird/tb/modules/exceptions.py +14 -0
- tinybird/tb/modules/feedback_manager.py +1 -1
- tinybird/tb/modules/info.py +69 -12
- tinybird/tb/modules/infra.py +4 -5
- tinybird/tb/modules/job_common.py +15 -0
- tinybird/tb/modules/local.py +143 -23
- tinybird/tb/modules/local_common.py +347 -19
- tinybird/tb/modules/local_logs.py +209 -0
- tinybird/tb/modules/login.py +21 -2
- tinybird/tb/modules/login_common.py +254 -12
- tinybird/tb/modules/mock.py +5 -54
- tinybird/tb/modules/mock_common.py +0 -54
- tinybird/tb/modules/open.py +10 -5
- tinybird/tb/modules/project.py +14 -5
- tinybird/tb/modules/shell.py +15 -7
- tinybird/tb/modules/sink.py +3 -1
- tinybird/tb/modules/telemetry.py +11 -3
- tinybird/tb/modules/test.py +13 -9
- tinybird/tb/modules/test_common.py +13 -87
- tinybird/tb/modules/tinyunit/tinyunit.py +0 -14
- tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -6
- tinybird/tb/modules/watch.py +5 -3
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tb_cli_modules/telemetry.py +1 -1
- tinybird/tornado_template.py +6 -7
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/METADATA +32 -6
- tinybird-1.0.5.dist-info/RECORD +132 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/WHEEL +1 -1
- tinybird-0.0.1.dev291.dist-info/RECORD +0 -128
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/top_level.txt +0 -0
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
from typing import Any, Dict, List
|
|
2
|
-
|
|
3
|
-
from tinybird.prompts import mock_prompt
|
|
4
1
|
from tinybird.tb.client import TinyB
|
|
5
2
|
from tinybird.tb.modules.common import push_data
|
|
6
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
7
|
-
from tinybird.tb.modules.datafile.fixture import persist_fixture_sql
|
|
8
|
-
from tinybird.tb.modules.llm import LLM
|
|
9
|
-
from tinybird.tb.modules.llm_utils import extract_xml
|
|
10
3
|
|
|
11
4
|
|
|
12
5
|
def append_mock_data(
|
|
@@ -22,50 +15,3 @@ def append_mock_data(
|
|
|
22
15
|
concurrency=1,
|
|
23
16
|
silent=True,
|
|
24
17
|
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def create_mock_data(
|
|
28
|
-
datasource_name: str,
|
|
29
|
-
datasource_content: str,
|
|
30
|
-
rows: int,
|
|
31
|
-
prompt: str,
|
|
32
|
-
config: CLIConfig,
|
|
33
|
-
ctx_config: Dict[str, Any],
|
|
34
|
-
user_token: str,
|
|
35
|
-
tb_client: TinyB,
|
|
36
|
-
format_: str,
|
|
37
|
-
folder: str,
|
|
38
|
-
) -> List[Dict[str, Any]]:
|
|
39
|
-
user_client = config.get_client(token=ctx_config.get("token"), host=ctx_config.get("host"))
|
|
40
|
-
llm = LLM(user_token=user_token, host=user_client.host)
|
|
41
|
-
prompt = f"<datasource_schema>{datasource_content}</datasource_schema>\n<user_input>{prompt}</user_input>"
|
|
42
|
-
sql = ""
|
|
43
|
-
attempts = 0
|
|
44
|
-
data = []
|
|
45
|
-
error = ""
|
|
46
|
-
sql_path = None
|
|
47
|
-
while True:
|
|
48
|
-
try:
|
|
49
|
-
response = llm.ask(system_prompt=mock_prompt(rows, error), prompt=prompt, feature="tb_mock")
|
|
50
|
-
sql = extract_xml(response, "sql")
|
|
51
|
-
sql_path = persist_fixture_sql(datasource_name, sql, folder)
|
|
52
|
-
sql_format = "JSON" if format_ == "ndjson" else "CSV"
|
|
53
|
-
result = tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
|
|
54
|
-
if sql_format == "JSON":
|
|
55
|
-
data = result.get("data", [])[:rows]
|
|
56
|
-
error_response = result.get("error", None)
|
|
57
|
-
if error_response:
|
|
58
|
-
raise Exception(error_response)
|
|
59
|
-
else:
|
|
60
|
-
data = result
|
|
61
|
-
break
|
|
62
|
-
except Exception as e:
|
|
63
|
-
error = str(e)
|
|
64
|
-
attempts += 1
|
|
65
|
-
if attempts > 5:
|
|
66
|
-
raise Exception(
|
|
67
|
-
f"Failed to generate a valid solution. Check {str(sql_path or '.sql path')} and try again."
|
|
68
|
-
)
|
|
69
|
-
else:
|
|
70
|
-
continue
|
|
71
|
-
return data
|
tinybird/tb/modules/open.py
CHANGED
|
@@ -7,7 +7,6 @@ from tinybird.tb.config import get_display_cloud_host
|
|
|
7
7
|
from tinybird.tb.modules.cli import cli
|
|
8
8
|
from tinybird.tb.modules.exceptions import CLIException
|
|
9
9
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
10
|
-
from tinybird.tb.modules.local_common import get_build_workspace_name
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
@cli.command()
|
|
@@ -21,12 +20,12 @@ def open(ctx: Context, workspace: str):
|
|
|
21
20
|
|
|
22
21
|
config = ctx.ensure_object(dict)["config"]
|
|
23
22
|
client = ctx.ensure_object(dict)["client"]
|
|
24
|
-
|
|
23
|
+
branch = ctx.ensure_object(dict)["branch"]
|
|
25
24
|
|
|
26
25
|
url_host = get_display_cloud_host(client.host)
|
|
27
26
|
|
|
28
27
|
if not workspace:
|
|
29
|
-
workspace =
|
|
28
|
+
workspace = config.get("name")
|
|
30
29
|
|
|
31
30
|
if not workspace:
|
|
32
31
|
raise CLIException(
|
|
@@ -35,7 +34,13 @@ def open(ctx: Context, workspace: str):
|
|
|
35
34
|
)
|
|
36
35
|
)
|
|
37
36
|
|
|
38
|
-
|
|
37
|
+
if branch:
|
|
38
|
+
click.echo(
|
|
39
|
+
FeedbackManager.highlight(message=f"» Opening branch {branch} of workspace {workspace} in the browser")
|
|
40
|
+
)
|
|
41
|
+
auth_url = f"{url_host}/{workspace}~{branch}"
|
|
42
|
+
else:
|
|
43
|
+
click.echo(FeedbackManager.highlight(message=f"» Opening workspace {workspace} in the browser"))
|
|
44
|
+
auth_url = f"{url_host}/{workspace}"
|
|
39
45
|
|
|
40
|
-
auth_url = f"{url_host}/{workspace}"
|
|
41
46
|
webbrowser.open(auth_url)
|
tinybird/tb/modules/project.py
CHANGED
|
@@ -130,7 +130,7 @@ class Project:
|
|
|
130
130
|
|
|
131
131
|
@property
|
|
132
132
|
def connections(self) -> List[str]:
|
|
133
|
-
return sorted([Path(f).stem for f in self.
|
|
133
|
+
return sorted([Path(f).stem for f in self._get_connection_files()])
|
|
134
134
|
|
|
135
135
|
def get_datasource_files(self) -> List[str]:
|
|
136
136
|
return self.get_files("datasource")
|
|
@@ -138,17 +138,26 @@ class Project:
|
|
|
138
138
|
def get_pipe_files(self) -> List[str]:
|
|
139
139
|
return self.get_files("pipe")
|
|
140
140
|
|
|
141
|
-
def
|
|
141
|
+
def _get_connection_files(self) -> List[str]:
|
|
142
142
|
return self.get_files("connection")
|
|
143
143
|
|
|
144
|
+
def get_connection_files(self, connection_type: Optional[str] = None) -> List[str]:
|
|
145
|
+
if connection_type == "kafka":
|
|
146
|
+
return self.get_kafka_connection_files()
|
|
147
|
+
if connection_type == "s3":
|
|
148
|
+
return self.get_s3_connection_files()
|
|
149
|
+
if connection_type == "gcs":
|
|
150
|
+
return self.get_gcs_connection_files()
|
|
151
|
+
return self._get_connection_files()
|
|
152
|
+
|
|
144
153
|
def get_kafka_connection_files(self) -> List[str]:
|
|
145
|
-
return [f for f in self.
|
|
154
|
+
return [f for f in self._get_connection_files() if self.is_kafka_connection(Path(f).read_text())]
|
|
146
155
|
|
|
147
156
|
def get_s3_connection_files(self) -> List[str]:
|
|
148
|
-
return [f for f in self.
|
|
157
|
+
return [f for f in self._get_connection_files() if self.is_s3_connection(Path(f).read_text())]
|
|
149
158
|
|
|
150
159
|
def get_gcs_connection_files(self) -> List[str]:
|
|
151
|
-
return [f for f in self.
|
|
160
|
+
return [f for f in self._get_connection_files() if self.is_gcs_connection(Path(f).read_text())]
|
|
152
161
|
|
|
153
162
|
def get_pipe_datafile(self, filename: str) -> Optional[Datafile]:
|
|
154
163
|
try:
|
tinybird/tb/modules/shell.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import subprocess
|
|
3
3
|
import sys
|
|
4
|
-
from typing import List
|
|
4
|
+
from typing import List, Optional
|
|
5
5
|
|
|
6
6
|
import click
|
|
7
7
|
import humanfriendly
|
|
@@ -192,11 +192,14 @@ def _(event):
|
|
|
192
192
|
|
|
193
193
|
|
|
194
194
|
class Shell:
|
|
195
|
-
def __init__(self, project: Project, tb_client: TinyB, playground:
|
|
195
|
+
def __init__(self, project: Project, tb_client: TinyB, playground=False, branch: Optional[str] = None):
|
|
196
196
|
self.history = self.get_history()
|
|
197
197
|
self.project = project
|
|
198
198
|
self.tb_client = tb_client
|
|
199
|
-
|
|
199
|
+
if playground:
|
|
200
|
+
self.env = "--cloud"
|
|
201
|
+
else:
|
|
202
|
+
self.env = f"--branch={branch}" if branch else "--local"
|
|
200
203
|
self.prompt_message = "\ntb » "
|
|
201
204
|
self.session: PromptSession = PromptSession(
|
|
202
205
|
completer=DynamicCompleter(project),
|
|
@@ -245,6 +248,8 @@ class Shell:
|
|
|
245
248
|
self.handle_auth()
|
|
246
249
|
elif cmd == "workspace":
|
|
247
250
|
self.handle_workspace()
|
|
251
|
+
elif cmd == "branch":
|
|
252
|
+
self.handle_branch()
|
|
248
253
|
elif cmd == "deploy":
|
|
249
254
|
self.handle_deploy()
|
|
250
255
|
elif cmd == "mock":
|
|
@@ -264,13 +269,16 @@ class Shell:
|
|
|
264
269
|
def handle_workspace(self):
|
|
265
270
|
click.echo(FeedbackManager.error(message="'tb workspace' is not available in the dev shell"))
|
|
266
271
|
|
|
272
|
+
def handle_branch(self):
|
|
273
|
+
click.echo(FeedbackManager.error(message="'tb branch' is not available in the dev shell"))
|
|
274
|
+
|
|
267
275
|
def handle_deploy(self):
|
|
268
276
|
click.echo(FeedbackManager.error(message="'tb deploy' is not available in the dev shell"))
|
|
269
277
|
|
|
270
278
|
def handle_mock(self, arg):
|
|
271
279
|
if "mock" in arg.strip().lower():
|
|
272
280
|
arg = arg.replace("mock", "")
|
|
273
|
-
subprocess.run(f"tb
|
|
281
|
+
subprocess.run(f"tb {self.env} mock {arg}", shell=True, text=True)
|
|
274
282
|
|
|
275
283
|
def handle_tb(self, argline):
|
|
276
284
|
click.echo("")
|
|
@@ -287,14 +295,14 @@ class Shell:
|
|
|
287
295
|
need_skip = ("mock", "test create", "create")
|
|
288
296
|
if any(arg.startswith(cmd) for cmd in need_skip):
|
|
289
297
|
argline = f"{argline}"
|
|
290
|
-
subprocess.run(f"tb
|
|
298
|
+
subprocess.run(f"tb {self.env} {argline}", shell=True, text=True)
|
|
291
299
|
|
|
292
300
|
def default(self, argline):
|
|
293
301
|
click.echo("")
|
|
294
302
|
arg = argline.strip().lower()
|
|
295
303
|
if not arg:
|
|
296
304
|
return
|
|
297
|
-
if arg.startswith("with"
|
|
305
|
+
if arg.startswith(("with", "select")):
|
|
298
306
|
self.run_sql(argline)
|
|
299
307
|
elif len(arg.split()) == 1 and arg in self.project.pipes + self.project.datasources:
|
|
300
308
|
self.run_sql(f"select * from {argline}")
|
|
@@ -302,7 +310,7 @@ class Shell:
|
|
|
302
310
|
need_skip = ("mock", "test create", "create")
|
|
303
311
|
if any(arg.startswith(cmd) for cmd in need_skip):
|
|
304
312
|
argline = f"{argline}"
|
|
305
|
-
subprocess.run(f"tb
|
|
313
|
+
subprocess.run(f"tb {self.env} {argline}", shell=True, text=True)
|
|
306
314
|
|
|
307
315
|
def run_sql(self, query, rows_limit=20):
|
|
308
316
|
try:
|
tinybird/tb/modules/sink.py
CHANGED
|
@@ -11,6 +11,7 @@ from tinybird.tb.modules.cli import cli
|
|
|
11
11
|
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table, wait_job
|
|
12
12
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
13
13
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
14
|
+
from tinybird.tb.modules.job_common import echo_job_url
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
@cli.group()
|
|
@@ -84,12 +85,13 @@ def sink_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, pa
|
|
|
84
85
|
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
85
86
|
click.echo(FeedbackManager.highlight(message=f"\n» Running sink '{pipe_name_or_id}'"))
|
|
86
87
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
88
|
+
config = ctx.ensure_object(dict)["config"]
|
|
87
89
|
|
|
88
90
|
try:
|
|
89
91
|
response = client.pipe_run(pipe_name_or_id, "sink", params, mode)
|
|
90
92
|
job_id = response["job"]["id"]
|
|
91
93
|
job_url = response["job"]["job_url"]
|
|
92
|
-
|
|
94
|
+
echo_job_url(client.token, client.host, config.get("name") or "", job_url)
|
|
93
95
|
click.echo(FeedbackManager.success(message="✓ Sink job created"))
|
|
94
96
|
|
|
95
97
|
if wait:
|
tinybird/tb/modules/telemetry.py
CHANGED
|
@@ -42,8 +42,11 @@ def get_ci_product_name() -> Optional[str]:
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
def is_ci_environment() -> bool:
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
try:
|
|
46
|
+
ci_product: Optional[str] = get_ci_product_name()
|
|
47
|
+
return ci_product is not None
|
|
48
|
+
except Exception:
|
|
49
|
+
return False
|
|
47
50
|
|
|
48
51
|
|
|
49
52
|
def silence_errors(f: Callable) -> Callable:
|
|
@@ -86,7 +89,7 @@ class TelemetryHelper:
|
|
|
86
89
|
tb_host: Optional[str] = None,
|
|
87
90
|
max_enqueued_events: int = 5,
|
|
88
91
|
) -> None:
|
|
89
|
-
self.tb_host = tb_host or os.getenv("TB_CLI_TELEMETRY_HOST", "https://api.tinybird.co")
|
|
92
|
+
self.tb_host = tb_host or os.getenv("TB_CLI_TELEMETRY_HOST", "https://api.europe-west2.gcp.tinybird.co")
|
|
90
93
|
self.max_enqueued_events: int = max_enqueued_events
|
|
91
94
|
|
|
92
95
|
self.enabled: bool = True
|
|
@@ -133,6 +136,7 @@ class TelemetryHelper:
|
|
|
133
136
|
event_dict["event"] = event
|
|
134
137
|
event_dict["event_data"] = json.dumps(event_data)
|
|
135
138
|
event_dict["timestamp"] = datetime.utcnow().isoformat()
|
|
139
|
+
event_dict["cli"] = "forward"
|
|
136
140
|
|
|
137
141
|
self.events.append(event_dict)
|
|
138
142
|
if len(self.events) >= self.max_enqueued_events:
|
|
@@ -284,6 +288,10 @@ def add_telemetry_sysinfo_event() -> None:
|
|
|
284
288
|
|
|
285
289
|
cli_args = sys.argv[1:] if len(sys.argv) > 1 else []
|
|
286
290
|
|
|
291
|
+
# we don't track login commands because we track them explicitly
|
|
292
|
+
if "login" in cli_args:
|
|
293
|
+
return
|
|
294
|
+
|
|
287
295
|
is_secret_command = "secret" in cli_args
|
|
288
296
|
if is_secret_command:
|
|
289
297
|
need_to_obfuscate = "set" in cli_args
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -3,14 +3,15 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
from typing import Tuple
|
|
6
|
+
from typing import Any, Tuple
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
|
|
10
10
|
from tinybird.tb.client import TinyB
|
|
11
|
+
from tinybird.tb.modules.agent import run_agent
|
|
11
12
|
from tinybird.tb.modules.cli import cli
|
|
12
13
|
from tinybird.tb.modules.project import Project
|
|
13
|
-
from tinybird.tb.modules.test_common import
|
|
14
|
+
from tinybird.tb.modules.test_common import run_tests, update_test
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
@cli.group()
|
|
@@ -24,17 +25,18 @@ def test(ctx: click.Context) -> None:
|
|
|
24
25
|
help="Create a test for an existing pipe",
|
|
25
26
|
)
|
|
26
27
|
@click.argument("name_or_filename", type=str)
|
|
27
|
-
@click.option(
|
|
28
|
-
"--prompt", type=str, default="Create a test for the selected pipe", help="Prompt to be used to create the test"
|
|
29
|
-
)
|
|
28
|
+
@click.option("--prompt", type=str, default="", help="Prompt to be used to create the test")
|
|
30
29
|
@click.pass_context
|
|
31
30
|
def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
32
31
|
"""
|
|
33
32
|
Create a test for an existing pipe
|
|
34
33
|
"""
|
|
35
34
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
36
|
+
prompt = (
|
|
37
|
+
f"""Create tests for the following pipe: {name_or_filename}. Extra context: {prompt or "No extra context."}"""
|
|
38
|
+
)
|
|
39
|
+
run_agent(config, project, True, prompt=prompt, feature="tb_test_create")
|
|
38
40
|
|
|
39
41
|
|
|
40
42
|
@test.command(
|
|
@@ -46,7 +48,8 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
46
48
|
def test_update(ctx: click.Context, pipe: str) -> None:
|
|
47
49
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
48
50
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
49
|
-
|
|
51
|
+
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
52
|
+
update_test(pipe, project, client, config=config)
|
|
50
53
|
|
|
51
54
|
|
|
52
55
|
@test.command(
|
|
@@ -58,4 +61,5 @@ def test_update(ctx: click.Context, pipe: str) -> None:
|
|
|
58
61
|
def run_tests_command(ctx: click.Context, name: Tuple[str, ...]) -> None:
|
|
59
62
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
60
63
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
61
|
-
|
|
64
|
+
config: dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
65
|
+
run_tests(name, project, client, config=config)
|
|
@@ -14,15 +14,11 @@ import click
|
|
|
14
14
|
import yaml
|
|
15
15
|
from requests import Response
|
|
16
16
|
|
|
17
|
-
from tinybird.prompts import test_create_prompt
|
|
18
17
|
from tinybird.tb.client import TinyB
|
|
19
18
|
from tinybird.tb.modules.build_common import process as build_project
|
|
20
19
|
from tinybird.tb.modules.common import sys_exit
|
|
21
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
22
20
|
from tinybird.tb.modules.exceptions import CLITestException
|
|
23
21
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
24
|
-
from tinybird.tb.modules.llm import LLM
|
|
25
|
-
from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
|
|
26
22
|
from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
|
|
27
23
|
from tinybird.tb.modules.project import Project
|
|
28
24
|
|
|
@@ -54,85 +50,6 @@ def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Opti
|
|
|
54
50
|
return path
|
|
55
51
|
|
|
56
52
|
|
|
57
|
-
def create_test(
|
|
58
|
-
name_or_filename: str, prompt: str, project: Project, client: TinyB, preview: bool = False
|
|
59
|
-
) -> list[dict[str, Any]]:
|
|
60
|
-
"""
|
|
61
|
-
Create a test for an existing pipe
|
|
62
|
-
"""
|
|
63
|
-
tests: List[Dict[str, Any]] = []
|
|
64
|
-
|
|
65
|
-
try:
|
|
66
|
-
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
67
|
-
build_error = build_project(project=project, tb_client=client, watch=False, silent=True, exit_on_error=False)
|
|
68
|
-
if build_error:
|
|
69
|
-
raise Exception(build_error)
|
|
70
|
-
click.echo(FeedbackManager.info(message="✓ Done!\n"))
|
|
71
|
-
config = CLIConfig.get_project_config()
|
|
72
|
-
folder = project.folder
|
|
73
|
-
pipe_path = get_pipe_path(name_or_filename, folder)
|
|
74
|
-
pipe_name = pipe_path.stem
|
|
75
|
-
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
76
|
-
pipe_content = pipe_path.read_text()
|
|
77
|
-
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
78
|
-
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
79
|
-
|
|
80
|
-
system_prompt = test_create_prompt.format(
|
|
81
|
-
name=pipe_name,
|
|
82
|
-
content=pipe_content,
|
|
83
|
-
parameters=parameters or "No parameters",
|
|
84
|
-
)
|
|
85
|
-
user_token = config.get_user_token()
|
|
86
|
-
if not user_token:
|
|
87
|
-
raise Exception("No user token found")
|
|
88
|
-
|
|
89
|
-
llm = LLM(user_token=user_token, host=config.get_client().host)
|
|
90
|
-
response_llm = llm.ask(system_prompt=system_prompt, prompt=prompt, feature="tb_test_create")
|
|
91
|
-
response_xml = extract_xml(response_llm, "response")
|
|
92
|
-
tests_content = parse_xml(response_xml, "test")
|
|
93
|
-
|
|
94
|
-
for test_content in tests_content:
|
|
95
|
-
test: Dict[str, Any] = {}
|
|
96
|
-
test["name"] = extract_xml(test_content, "name")
|
|
97
|
-
test["description"] = extract_xml(test_content, "description")
|
|
98
|
-
parameters_api = extract_xml(test_content, "parameters")
|
|
99
|
-
test["parameters"] = parameters_api.split("?")[1] if "?" in parameters_api else parameters_api
|
|
100
|
-
test["expected_result"] = ""
|
|
101
|
-
|
|
102
|
-
response = None
|
|
103
|
-
try:
|
|
104
|
-
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
105
|
-
except Exception:
|
|
106
|
-
pass
|
|
107
|
-
|
|
108
|
-
if response:
|
|
109
|
-
if response.status_code >= 400:
|
|
110
|
-
test["expected_http_status"] = response.status_code
|
|
111
|
-
test["expected_result"] = response.json()["error"]
|
|
112
|
-
else:
|
|
113
|
-
test.pop("expected_http_status", None)
|
|
114
|
-
test["expected_result"] = response.text or ""
|
|
115
|
-
|
|
116
|
-
tests.append(test)
|
|
117
|
-
|
|
118
|
-
if not preview:
|
|
119
|
-
if len(tests) > 0:
|
|
120
|
-
generate_test_file(pipe_name, tests, folder, mode="a")
|
|
121
|
-
for test in tests:
|
|
122
|
-
test_name = test["name"]
|
|
123
|
-
click.echo(FeedbackManager.info(message=f"✓ {test_name} created"))
|
|
124
|
-
else:
|
|
125
|
-
click.echo(FeedbackManager.info(message="* No tests created"))
|
|
126
|
-
|
|
127
|
-
click.echo(FeedbackManager.success(message="✓ Done!\n"))
|
|
128
|
-
except Exception as e:
|
|
129
|
-
raise CLITestException(FeedbackManager.error(message=str(e)))
|
|
130
|
-
finally:
|
|
131
|
-
cleanup_test_workspace(client, project.folder)
|
|
132
|
-
|
|
133
|
-
return tests
|
|
134
|
-
|
|
135
|
-
|
|
136
53
|
def parse_tests(tests_content: str) -> List[Dict[str, Any]]:
|
|
137
54
|
return yaml.safe_load(tests_content)
|
|
138
55
|
|
|
@@ -142,11 +59,13 @@ def dump_tests(tests: List[Dict[str, Any]]) -> str:
|
|
|
142
59
|
return yaml_str.replace("- name:", "\n- name:")
|
|
143
60
|
|
|
144
61
|
|
|
145
|
-
def update_test(pipe: str, project: Project, client: TinyB) -> None:
|
|
62
|
+
def update_test(pipe: str, project: Project, client: TinyB, config: dict[str, Any]) -> None:
|
|
146
63
|
try:
|
|
147
64
|
folder = project.folder
|
|
148
65
|
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
149
|
-
build_error = build_project(
|
|
66
|
+
build_error = build_project(
|
|
67
|
+
project=project, tb_client=client, watch=False, silent=True, exit_on_error=False, config=config
|
|
68
|
+
)
|
|
150
69
|
if build_error:
|
|
151
70
|
raise Exception(build_error)
|
|
152
71
|
|
|
@@ -190,11 +109,18 @@ def update_test(pipe: str, project: Project, client: TinyB) -> None:
|
|
|
190
109
|
cleanup_test_workspace(client, project.folder)
|
|
191
110
|
|
|
192
111
|
|
|
193
|
-
def run_tests(name: Tuple[str, ...], project: Project, client: TinyB) -> Optional[str]:
|
|
112
|
+
def run_tests(name: Tuple[str, ...], project: Project, client: TinyB, config: dict[str, Any]) -> Optional[str]:
|
|
194
113
|
full_error = ""
|
|
195
114
|
try:
|
|
196
115
|
click.echo(FeedbackManager.highlight(message="\n» Building test environment"))
|
|
197
|
-
build_error = build_project(
|
|
116
|
+
build_error = build_project(
|
|
117
|
+
project=project,
|
|
118
|
+
tb_client=client,
|
|
119
|
+
watch=False,
|
|
120
|
+
silent=True,
|
|
121
|
+
exit_on_error=False,
|
|
122
|
+
config=config,
|
|
123
|
+
)
|
|
198
124
|
if build_error:
|
|
199
125
|
raise Exception(build_error)
|
|
200
126
|
click.echo(FeedbackManager.info(message="✓ Done!"))
|
|
@@ -5,7 +5,6 @@ from typing import Any, Dict, Iterable, List, Optional
|
|
|
5
5
|
import click
|
|
6
6
|
import yaml
|
|
7
7
|
from humanfriendly.tables import format_smart_table
|
|
8
|
-
from typing_extensions import override
|
|
9
8
|
|
|
10
9
|
from tinybird.tb.client import TinyB
|
|
11
10
|
from tinybird.tb.modules.common import CLIException
|
|
@@ -118,19 +117,6 @@ class TestResult:
|
|
|
118
117
|
return PASS_OVER_TIME
|
|
119
118
|
return PASS
|
|
120
119
|
|
|
121
|
-
@override
|
|
122
|
-
def __dict__(self):
|
|
123
|
-
return {
|
|
124
|
-
"name": self.name,
|
|
125
|
-
"data": self.data,
|
|
126
|
-
"elapsed_time": self.elapsed_time,
|
|
127
|
-
"read_bytes": self.read_bytes,
|
|
128
|
-
"max_elapsed_time": self.max_elapsed_time,
|
|
129
|
-
"max_bytes_read": self.max_bytes_read,
|
|
130
|
-
"error": self.error,
|
|
131
|
-
"status": self.status.name,
|
|
132
|
-
}
|
|
133
|
-
|
|
134
120
|
|
|
135
121
|
@dataclass()
|
|
136
122
|
class TestSummaryResults:
|
|
@@ -3,8 +3,6 @@ from collections import namedtuple
|
|
|
3
3
|
from json import JSONEncoder
|
|
4
4
|
from typing import Optional
|
|
5
5
|
|
|
6
|
-
from typing_extensions import override
|
|
7
|
-
|
|
8
6
|
|
|
9
7
|
class MyJSONEncoder(JSONEncoder):
|
|
10
8
|
# def default(self, in_obj):
|
|
@@ -56,10 +54,6 @@ class DataUnitTest:
|
|
|
56
54
|
def __str__(self):
|
|
57
55
|
return json.dumps(dict(self), ensure_ascii=False)
|
|
58
56
|
|
|
59
|
-
@override
|
|
60
|
-
def __dict__(self):
|
|
61
|
-
return dict(self)
|
|
62
|
-
|
|
63
57
|
def __repr__(self):
|
|
64
58
|
return self.__str__()
|
|
65
59
|
|
tinybird/tb/modules/watch.py
CHANGED
|
@@ -33,9 +33,10 @@ class WatchProjectHandler(PatternMatchingEventHandler):
|
|
|
33
33
|
".env.local",
|
|
34
34
|
]
|
|
35
35
|
|
|
36
|
-
def __init__(self, shell: Shell, project: Project, process: Callable):
|
|
36
|
+
def __init__(self, shell: Shell, project: Project, config: dict[str, Any], process: Callable):
|
|
37
37
|
self.shell = shell
|
|
38
38
|
self.project = project
|
|
39
|
+
self.config = config
|
|
39
40
|
self.process = process
|
|
40
41
|
self.datafiles = project.get_project_datafiles()
|
|
41
42
|
patterns = [f"**/*{ext}" for ext in self.valid_extensions]
|
|
@@ -58,7 +59,7 @@ class WatchProjectHandler(PatternMatchingEventHandler):
|
|
|
58
59
|
|
|
59
60
|
def _process(self, path: Optional[str] = None) -> None:
|
|
60
61
|
click.echo(FeedbackManager.highlight(message="» Rebuilding project..."))
|
|
61
|
-
self.process(watch=True, file_changed=path, diff=self.diff(path))
|
|
62
|
+
self.process(watch=True, file_changed=path, diff=self.diff(path), config=self.config)
|
|
62
63
|
self.shell.reprint_prompt()
|
|
63
64
|
|
|
64
65
|
def diff(self, path: Optional[str] = None) -> Optional[str]:
|
|
@@ -137,8 +138,9 @@ def watch_project(
|
|
|
137
138
|
shell: Shell,
|
|
138
139
|
process: Callable[[bool, Optional[str], Optional[str]], None],
|
|
139
140
|
project: Project,
|
|
141
|
+
config: dict[str, Any],
|
|
140
142
|
) -> None:
|
|
141
|
-
event_handler = WatchProjectHandler(shell=shell, project=project, process=process)
|
|
143
|
+
event_handler = WatchProjectHandler(shell=shell, project=project, process=process, config=config)
|
|
142
144
|
observer = Observer()
|
|
143
145
|
observer.schedule(event_handler, path=str(project.path), recursive=True)
|
|
144
146
|
observer.start()
|
|
@@ -1042,7 +1042,7 @@ def get_format_from_filename_or_url(filename_or_url: str) -> str:
|
|
|
1042
1042
|
'csv'
|
|
1043
1043
|
"""
|
|
1044
1044
|
filename_or_url = filename_or_url.lower()
|
|
1045
|
-
if filename_or_url.endswith("json"
|
|
1045
|
+
if filename_or_url.endswith(("json", "ndjson")):
|
|
1046
1046
|
return "ndjson"
|
|
1047
1047
|
if filename_or_url.endswith("parquet"):
|
|
1048
1048
|
return "parquet"
|
|
@@ -1050,7 +1050,7 @@ def get_format_from_filename_or_url(filename_or_url: str) -> str:
|
|
|
1050
1050
|
return "csv"
|
|
1051
1051
|
try:
|
|
1052
1052
|
parsed = urlparse(filename_or_url)
|
|
1053
|
-
if parsed.path.endswith("json"
|
|
1053
|
+
if parsed.path.endswith(("json", "ndjson")):
|
|
1054
1054
|
return "ndjson"
|
|
1055
1055
|
if parsed.path.endswith("parquet"):
|
|
1056
1056
|
return "parquet"
|
|
@@ -81,7 +81,7 @@ def _hide_tokens(text: str) -> str:
|
|
|
81
81
|
|
|
82
82
|
class TelemetryHelper:
|
|
83
83
|
def __init__(self, tb_host: Optional[str] = None, max_enqueued_events: int = 5) -> None:
|
|
84
|
-
self.tb_host = tb_host or os.getenv("TB_CLI_TELEMETRY_HOST", "https://api.tinybird.co")
|
|
84
|
+
self.tb_host = tb_host or os.getenv("TB_CLI_TELEMETRY_HOST", "https://api.europe-west2.gcp.tinybird.co")
|
|
85
85
|
self.max_enqueued_events: int = max_enqueued_events
|
|
86
86
|
|
|
87
87
|
self.enabled: bool = True
|
tinybird/tornado_template.py
CHANGED
|
@@ -289,12 +289,11 @@ class Template:
|
|
|
289
289
|
if whitespace is None:
|
|
290
290
|
if loader and loader.whitespace:
|
|
291
291
|
whitespace = loader.whitespace
|
|
292
|
-
|
|
292
|
+
elif name.endswith((".html", ".js")):
|
|
293
293
|
# Whitespace defaults by filename.
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
whitespace = "all"
|
|
294
|
+
whitespace = "single"
|
|
295
|
+
else:
|
|
296
|
+
whitespace = "all"
|
|
298
297
|
# Validate the whitespace setting.
|
|
299
298
|
filter_whitespace(whitespace, "")
|
|
300
299
|
|
|
@@ -1169,7 +1168,7 @@ def check_valid_expr(expr):
|
|
|
1169
1168
|
check_valid_expr(expr.slice.lower)
|
|
1170
1169
|
if expr.slice.upper is not None:
|
|
1171
1170
|
check_valid_expr(expr.slice.upper)
|
|
1172
|
-
elif isinstance(expr.slice, ast.Constant
|
|
1171
|
+
elif isinstance(expr.slice, (ast.Constant, ast.Subscript)):
|
|
1173
1172
|
check_valid_expr(expr.slice)
|
|
1174
1173
|
else:
|
|
1175
1174
|
raise SecurityException(f"Invalid Slice expression: {ast.dump(expr.slice)}")
|
|
@@ -1178,7 +1177,7 @@ def check_valid_expr(expr):
|
|
|
1178
1177
|
check_valid_expr(key)
|
|
1179
1178
|
for value in expr.values:
|
|
1180
1179
|
check_valid_expr(value)
|
|
1181
|
-
elif isinstance(expr, ast.Tuple
|
|
1180
|
+
elif isinstance(expr, (ast.Tuple, ast.List, ast.Set)):
|
|
1182
1181
|
for x in expr.elts:
|
|
1183
1182
|
check_valid_expr(x)
|
|
1184
1183
|
elif isinstance(expr, ast.JoinedStr):
|