tinybird 0.0.1.dev18__py3-none-any.whl → 0.0.1.dev20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/client.py +24 -4
- tinybird/config.py +1 -1
- tinybird/feedback_manager.py +8 -30
- tinybird/tb/__cli__.py +8 -0
- tinybird/tb/modules/auth.py +1 -1
- tinybird/tb/modules/build.py +26 -80
- tinybird/tb/modules/cicd.py +1 -1
- tinybird/tb/modules/cli.py +11 -837
- tinybird/tb/modules/common.py +1 -55
- tinybird/tb/modules/connection.py +1 -1
- tinybird/tb/modules/create.py +18 -7
- tinybird/tb/modules/datafile/build.py +142 -971
- tinybird/tb/modules/datafile/build_common.py +1 -1
- tinybird/tb/modules/datafile/build_datasource.py +1 -1
- tinybird/tb/modules/datafile/build_pipe.py +1 -1
- tinybird/tb/modules/datafile/common.py +12 -11
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/fixture.py +1 -1
- tinybird/tb/modules/datafile/format_common.py +0 -7
- tinybird/tb/modules/datafile/format_datasource.py +0 -2
- tinybird/tb/modules/datafile/format_pipe.py +0 -2
- tinybird/tb/modules/datafile/parse_datasource.py +1 -1
- tinybird/tb/modules/datafile/parse_pipe.py +1 -1
- tinybird/tb/modules/datafile/pull.py +1 -1
- tinybird/tb/modules/datasource.py +4 -75
- tinybird/tb/modules/feedback_manager.py +1048 -0
- tinybird/tb/modules/fmt.py +1 -1
- tinybird/tb/modules/job.py +1 -1
- tinybird/tb/modules/llm.py +6 -4
- tinybird/tb/modules/local.py +26 -21
- tinybird/tb/modules/local_common.py +2 -1
- tinybird/tb/modules/login.py +18 -11
- tinybird/tb/modules/mock.py +18 -7
- tinybird/tb/modules/pipe.py +4 -126
- tinybird/tb/modules/{build_shell.py → shell.py} +66 -36
- tinybird/tb/modules/table.py +88 -5
- tinybird/tb/modules/tag.py +2 -2
- tinybird/tb/modules/test.py +45 -29
- tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
- tinybird/tb/modules/token.py +2 -2
- tinybird/tb/modules/watch.py +72 -0
- tinybird/tb/modules/workspace.py +1 -1
- tinybird/tb/modules/workspace_members.py +1 -1
- {tinybird-0.0.1.dev18.dist-info → tinybird-0.0.1.dev20.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev20.dist-info/RECORD +76 -0
- tinybird-0.0.1.dev18.dist-info/RECORD +0 -73
- {tinybird-0.0.1.dev18.dist-info → tinybird-0.0.1.dev20.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev18.dist-info → tinybird-0.0.1.dev20.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev18.dist-info → tinybird-0.0.1.dev20.dist-info}/top_level.txt +0 -0
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import concurrent.futures
|
|
3
3
|
import os
|
|
4
|
-
import random
|
|
5
4
|
import subprocess
|
|
6
5
|
import sys
|
|
7
6
|
from typing import List
|
|
@@ -16,16 +15,19 @@ from prompt_toolkit.shortcuts import CompleteStyle
|
|
|
16
15
|
from prompt_toolkit.styles import Style
|
|
17
16
|
|
|
18
17
|
from tinybird.client import TinyB
|
|
19
|
-
from tinybird.feedback_manager import FeedbackManager, bcolors
|
|
20
18
|
from tinybird.tb.modules.exceptions import CLIException
|
|
19
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager, bcolors
|
|
21
20
|
from tinybird.tb.modules.table import format_table
|
|
22
21
|
|
|
23
22
|
|
|
24
23
|
class DynamicCompleter(Completer):
|
|
25
|
-
def __init__(self, datasources: List[str], pipes: List[str]):
|
|
24
|
+
def __init__(self, datasources: List[str], shared_datasources: List[str], endpoints: List[str], pipes: List[str]):
|
|
26
25
|
self.datasources = datasources
|
|
26
|
+
self.shared_datasources = shared_datasources
|
|
27
|
+
self.endpoints = endpoints
|
|
27
28
|
self.pipes = pipes
|
|
28
29
|
self.static_commands = ["create", "mock", "test", "select"]
|
|
30
|
+
self.test_commands = ["create", "run", "update"]
|
|
29
31
|
self.mock_flags = ["--prompt", "--rows"]
|
|
30
32
|
self.common_rows = ["10", "50", "100", "500", "1000"]
|
|
31
33
|
self.sql_keywords = ["select", "from", "where", "group by", "order by", "limit"]
|
|
@@ -47,6 +49,8 @@ class DynamicCompleter(Completer):
|
|
|
47
49
|
|
|
48
50
|
if command == "mock":
|
|
49
51
|
yield from self._handle_mock_completions(words)
|
|
52
|
+
elif command == "test":
|
|
53
|
+
yield from self._handle_test_completions(words)
|
|
50
54
|
elif command == "select" or self._is_sql_query(text.lower()):
|
|
51
55
|
yield from self._handle_sql_completions(text)
|
|
52
56
|
else:
|
|
@@ -71,9 +75,9 @@ class DynamicCompleter(Completer):
|
|
|
71
75
|
if words[-1] == "from" or (
|
|
72
76
|
"from" in words and len(words) > words.index("from") + 1 and text_lower.endswith(" ")
|
|
73
77
|
):
|
|
74
|
-
for x in self.datasources:
|
|
78
|
+
for x in self.datasources + self.shared_datasources:
|
|
75
79
|
yield Completion(x, start_position=0, display=x, style="class:completion.datasource")
|
|
76
|
-
for x in self.pipes:
|
|
80
|
+
for x in self.endpoints + self.pipes:
|
|
77
81
|
yield Completion(x, start_position=0, display=x, style="class:completion.pipe")
|
|
78
82
|
return
|
|
79
83
|
|
|
@@ -89,22 +93,32 @@ class DynamicCompleter(Completer):
|
|
|
89
93
|
if len(words) == 1:
|
|
90
94
|
# After 'mock', show datasources
|
|
91
95
|
for ds in self.datasources:
|
|
92
|
-
yield Completion(ds, start_position=0, display=ds, style="class:completion.
|
|
96
|
+
yield Completion(ds, start_position=0, display=ds, style="class:completion.datasource")
|
|
93
97
|
return
|
|
94
98
|
|
|
95
99
|
if len(words) == 2 or len(words) == 4:
|
|
96
100
|
# After datasource or after a flag value, show available flags
|
|
97
101
|
available_flags = [f for f in self.mock_flags if f not in words]
|
|
98
102
|
for flag in available_flags:
|
|
99
|
-
yield Completion(flag, start_position=0, display=flag)
|
|
103
|
+
yield Completion(flag, start_position=0, display=flag, style="class:completion.cmd")
|
|
100
104
|
return
|
|
101
105
|
|
|
102
106
|
last_word = words[-1]
|
|
103
107
|
if last_word == "--prompt":
|
|
104
|
-
yield Completion('""', start_position=0, display='"Enter your prompt..."')
|
|
108
|
+
yield Completion('""', start_position=0, display='"Enter your prompt..."', style="class:completion.cmd")
|
|
105
109
|
elif last_word == "--rows":
|
|
106
110
|
for rows in self.common_rows:
|
|
107
|
-
yield Completion(rows, start_position=0, display=rows)
|
|
111
|
+
yield Completion(rows, start_position=0, display=rows, style="class:completion.cmd")
|
|
112
|
+
|
|
113
|
+
def _handle_test_completions(self, words: List[str]):
|
|
114
|
+
if len(words) == 1:
|
|
115
|
+
for cmd in self.test_commands:
|
|
116
|
+
yield Completion(cmd, start_position=0, display=cmd, style="class:completion.cmd")
|
|
117
|
+
return
|
|
118
|
+
elif len(words) == 2:
|
|
119
|
+
for cmd in self.endpoints:
|
|
120
|
+
yield Completion(cmd, start_position=0, display=cmd, style="class:completion.pipe")
|
|
121
|
+
return
|
|
108
122
|
|
|
109
123
|
def _yield_static_commands(self, current_word: str):
|
|
110
124
|
for cmd in self.static_commands:
|
|
@@ -116,6 +130,24 @@ class DynamicCompleter(Completer):
|
|
|
116
130
|
style="class:completion.cmd",
|
|
117
131
|
)
|
|
118
132
|
|
|
133
|
+
for cmd in self.datasources + self.shared_datasources:
|
|
134
|
+
if cmd.startswith(current_word):
|
|
135
|
+
yield Completion(
|
|
136
|
+
cmd,
|
|
137
|
+
start_position=-len(current_word) if current_word else 0,
|
|
138
|
+
display=cmd,
|
|
139
|
+
style="class:completion.datasource",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
for cmd in self.endpoints + self.pipes:
|
|
143
|
+
if cmd.startswith(current_word):
|
|
144
|
+
yield Completion(
|
|
145
|
+
cmd,
|
|
146
|
+
start_position=-len(current_word) if current_word else 0,
|
|
147
|
+
display=cmd,
|
|
148
|
+
style="class:completion.pipe",
|
|
149
|
+
)
|
|
150
|
+
|
|
119
151
|
|
|
120
152
|
style = Style.from_dict(
|
|
121
153
|
{
|
|
@@ -143,18 +175,27 @@ def _(event):
|
|
|
143
175
|
b.start_completion(select_first=False)
|
|
144
176
|
|
|
145
177
|
|
|
146
|
-
class
|
|
147
|
-
def __init__(
|
|
178
|
+
class Shell:
|
|
179
|
+
def __init__(
|
|
180
|
+
self,
|
|
181
|
+
folder: str,
|
|
182
|
+
client: TinyB,
|
|
183
|
+
datasources: List[str],
|
|
184
|
+
shared_datasources: List[str],
|
|
185
|
+
pipes: List[str],
|
|
186
|
+
endpoints: List[str],
|
|
187
|
+
):
|
|
148
188
|
self.history = self.get_history()
|
|
149
189
|
self.folder = folder
|
|
150
190
|
self.client = client
|
|
151
191
|
self.datasources = datasources
|
|
192
|
+
self.shared_datasources = shared_datasources
|
|
152
193
|
self.pipes = pipes
|
|
194
|
+
self.endpoints = endpoints
|
|
153
195
|
self.prompt_message = "\ntb > "
|
|
154
196
|
self.commands = ["create", "mock", "test", "tb", "select"]
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
completer=DynamicCompleter(self.datasources, self.pipes),
|
|
197
|
+
self.session: PromptSession = PromptSession(
|
|
198
|
+
completer=DynamicCompleter(self.datasources, self.shared_datasources, self.endpoints, self.pipes),
|
|
158
199
|
complete_style=CompleteStyle.COLUMN,
|
|
159
200
|
complete_while_typing=True,
|
|
160
201
|
history=self.history,
|
|
@@ -167,7 +208,7 @@ class BuildShell:
|
|
|
167
208
|
except Exception:
|
|
168
209
|
return None
|
|
169
210
|
|
|
170
|
-
def
|
|
211
|
+
def run(self):
|
|
171
212
|
while True:
|
|
172
213
|
try:
|
|
173
214
|
user_input = self.session.prompt(
|
|
@@ -241,10 +282,9 @@ class BuildShell:
|
|
|
241
282
|
if not arg:
|
|
242
283
|
return
|
|
243
284
|
if arg.startswith("with") or arg.startswith("select"):
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
click.echo(FeedbackManager.error(message=str(e)))
|
|
285
|
+
self.run_sql(argline)
|
|
286
|
+
elif len(arg.split()) == 1 and arg in self.endpoints + self.pipes + self.datasources + self.shared_datasources:
|
|
287
|
+
self.run_sql(f"select * from {arg}")
|
|
248
288
|
else:
|
|
249
289
|
subprocess.run(f"tb --local {arg}", shell=True, text=True)
|
|
250
290
|
|
|
@@ -271,10 +311,10 @@ class BuildShell:
|
|
|
271
311
|
res = executor.submit(run_query_in_thread).result()
|
|
272
312
|
|
|
273
313
|
except Exception as e:
|
|
274
|
-
|
|
314
|
+
click.echo(FeedbackManager.error_exception(error=str(e)))
|
|
275
315
|
|
|
276
316
|
if isinstance(res, dict) and "error" in res:
|
|
277
|
-
|
|
317
|
+
click.echo(FeedbackManager.error_exception(error=res["error"]))
|
|
278
318
|
|
|
279
319
|
if isinstance(res, dict) and "data" in res and res["data"]:
|
|
280
320
|
print_table_formatted(res, "QUERY")
|
|
@@ -286,9 +326,6 @@ class BuildShell:
|
|
|
286
326
|
|
|
287
327
|
|
|
288
328
|
def print_table_formatted(res: dict, name: str):
|
|
289
|
-
rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
|
|
290
|
-
rebuild_index = random.randint(0, len(rebuild_colors) - 1)
|
|
291
|
-
rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
|
|
292
329
|
data = []
|
|
293
330
|
limit = 5
|
|
294
331
|
for d in res["data"][:5]:
|
|
@@ -299,24 +336,17 @@ def print_table_formatted(res: dict, name: str):
|
|
|
299
336
|
elapsed = stats.get("elapsed", 0)
|
|
300
337
|
cols = len(meta)
|
|
301
338
|
try:
|
|
302
|
-
|
|
303
|
-
def print_message(message: str, color=bcolors.CGREY):
|
|
304
|
-
return f"{color}{message}{bcolors.ENDC}"
|
|
305
|
-
|
|
306
339
|
table = format_table(data, meta)
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
click.echo(
|
|
310
|
-
click.echo(colored_char)
|
|
311
|
-
click.echo(table_with_marker)
|
|
312
|
-
click.echo(colored_char)
|
|
340
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Running {name}\n"))
|
|
341
|
+
click.echo(table)
|
|
342
|
+
click.echo("")
|
|
313
343
|
rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
|
|
314
344
|
bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
|
|
315
345
|
elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
|
|
316
346
|
stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
|
|
317
347
|
rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
|
|
318
|
-
click.echo(
|
|
319
|
-
click.echo(
|
|
348
|
+
click.echo(FeedbackManager.success(message=stats_message))
|
|
349
|
+
click.echo(FeedbackManager.gray(message=rows_message))
|
|
320
350
|
except ValueError as exc:
|
|
321
351
|
if str(exc) == "max() arg is an empty sequence":
|
|
322
352
|
click.echo("------------")
|
tinybird/tb/modules/table.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Standard library modules.
|
|
2
2
|
import collections
|
|
3
3
|
import re
|
|
4
|
+
from typing import Union
|
|
4
5
|
|
|
5
6
|
# Modules included in our package.
|
|
6
7
|
from humanfriendly.compat import coerce_string
|
|
7
|
-
from humanfriendly.tables import format_robust_table
|
|
8
8
|
from humanfriendly.terminal import (
|
|
9
9
|
ansi_strip,
|
|
10
10
|
ansi_width,
|
|
@@ -13,7 +13,7 @@ from humanfriendly.terminal import (
|
|
|
13
13
|
terminal_supports_colors,
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
-
from tinybird.feedback_manager import bcolors
|
|
16
|
+
from tinybird.tb.modules.feedback_manager import bcolors
|
|
17
17
|
|
|
18
18
|
NUMERIC_DATA_PATTERN = re.compile(r"^\d+(\.\d+)?$")
|
|
19
19
|
|
|
@@ -133,7 +133,7 @@ def format_pretty_table(data, column_names=None, column_types=None, horizontal_b
|
|
|
133
133
|
column_types = [highlight_column_type(t) for t in column_types]
|
|
134
134
|
data.insert(1, column_types)
|
|
135
135
|
# Calculate the maximum width of each column.
|
|
136
|
-
widths = collections.defaultdict(int)
|
|
136
|
+
widths: dict[int, int] = collections.defaultdict(int)
|
|
137
137
|
numeric_data = collections.defaultdict(list)
|
|
138
138
|
for row_index, row in enumerate(data):
|
|
139
139
|
for column_index, column in enumerate(row):
|
|
@@ -177,9 +177,92 @@ def highlight_column_name(name):
|
|
|
177
177
|
return ansi_wrap(name)
|
|
178
178
|
|
|
179
179
|
|
|
180
|
+
def highlight_column_name_robust(name):
|
|
181
|
+
return f"{bcolors.CGREY}{name}{bcolors.ENDC}"
|
|
182
|
+
|
|
183
|
+
|
|
180
184
|
def highlight_column_type(type):
|
|
181
185
|
return f"{bcolors.CGREY}\033[3m{type}\033[23m{bcolors.ENDC}"
|
|
182
186
|
|
|
183
187
|
|
|
184
|
-
def highlight_horizontal_bar(horizontal_bar):
|
|
185
|
-
return f"
|
|
188
|
+
def highlight_horizontal_bar(horizontal_bar: str) -> str:
|
|
189
|
+
return f"\033[38;5;247m\033[3m{horizontal_bar}\033[23m{bcolors.ENDC}"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def format_robust_table(data, column_names):
|
|
193
|
+
"""
|
|
194
|
+
Render tabular data with one column per line (allowing columns with line breaks).
|
|
195
|
+
|
|
196
|
+
:param data: An iterable (e.g. a :func:`tuple` or :class:`list`)
|
|
197
|
+
containing the rows of the table, where each row is an
|
|
198
|
+
iterable containing the columns of the table (strings).
|
|
199
|
+
:param column_names: An iterable of column names (strings).
|
|
200
|
+
:returns: The rendered table (a string).
|
|
201
|
+
|
|
202
|
+
Here's an example:
|
|
203
|
+
|
|
204
|
+
>>> from humanfriendly.tables import format_robust_table
|
|
205
|
+
>>> column_names = ['Version', 'Uploaded on', 'Downloads']
|
|
206
|
+
>>> humanfriendly_releases = [
|
|
207
|
+
... ['1.23', '2015-05-25', '218'],
|
|
208
|
+
... ['1.23.1', '2015-05-26', '1354'],
|
|
209
|
+
... ['1.24', '2015-05-26', '223'],
|
|
210
|
+
... ['1.25', '2015-05-26', '4319'],
|
|
211
|
+
... ['1.25.1', '2015-06-02', '197'],
|
|
212
|
+
... ]
|
|
213
|
+
>>> print(format_robust_table(humanfriendly_releases, column_names))
|
|
214
|
+
-----------------------
|
|
215
|
+
Version: 1.23
|
|
216
|
+
Uploaded on: 2015-05-25
|
|
217
|
+
Downloads: 218
|
|
218
|
+
-----------------------
|
|
219
|
+
Version: 1.23.1
|
|
220
|
+
Uploaded on: 2015-05-26
|
|
221
|
+
Downloads: 1354
|
|
222
|
+
-----------------------
|
|
223
|
+
Version: 1.24
|
|
224
|
+
Uploaded on: 2015-05-26
|
|
225
|
+
Downloads: 223
|
|
226
|
+
-----------------------
|
|
227
|
+
Version: 1.25
|
|
228
|
+
Uploaded on: 2015-05-26
|
|
229
|
+
Downloads: 4319
|
|
230
|
+
-----------------------
|
|
231
|
+
Version: 1.25.1
|
|
232
|
+
Uploaded on: 2015-06-02
|
|
233
|
+
Downloads: 197
|
|
234
|
+
-----------------------
|
|
235
|
+
|
|
236
|
+
The column names are highlighted in bold font and color so they stand out a
|
|
237
|
+
bit more (see :data:`.HIGHLIGHT_COLOR`).
|
|
238
|
+
"""
|
|
239
|
+
blocks: list[Union[str, list[str]]] = []
|
|
240
|
+
column_names = ["%s:" % n for n in normalize_columns(column_names)]
|
|
241
|
+
if terminal_supports_colors():
|
|
242
|
+
column_names = [highlight_column_name_robust(n) for n in column_names]
|
|
243
|
+
# Convert each row into one or more `name: value' lines (one per column)
|
|
244
|
+
# and group each `row of lines' into a block (i.e. rows become blocks).
|
|
245
|
+
for row in data:
|
|
246
|
+
lines = []
|
|
247
|
+
for column_index, column_text in enumerate(normalize_columns(row)):
|
|
248
|
+
stripped_column = column_text.strip()
|
|
249
|
+
if "\n" not in stripped_column:
|
|
250
|
+
# Columns without line breaks are formatted inline.
|
|
251
|
+
lines.append("%s %s" % (column_names[column_index], stripped_column))
|
|
252
|
+
else:
|
|
253
|
+
# Columns with line breaks could very well contain indented
|
|
254
|
+
# lines, so we'll put the column name on a separate line. This
|
|
255
|
+
# way any indentation remains intact, and it's easier to
|
|
256
|
+
# copy/paste the text.
|
|
257
|
+
lines.append(column_names[column_index])
|
|
258
|
+
lines.extend(column_text.rstrip().splitlines())
|
|
259
|
+
blocks.append(lines)
|
|
260
|
+
# Calculate the width of the row delimiter.
|
|
261
|
+
num_rows, num_columns = find_terminal_size()
|
|
262
|
+
longest_line = max(max(map(ansi_width, lines)) for lines in blocks)
|
|
263
|
+
delimiter = highlight_horizontal_bar("\n%s\n" % ("─" * min(longest_line, num_columns)))
|
|
264
|
+
# Force a delimiter at the start and end of the table.
|
|
265
|
+
blocks.insert(0, "")
|
|
266
|
+
blocks.append("")
|
|
267
|
+
# Embed the row delimiter between every two blocks.
|
|
268
|
+
return delimiter.join("\n".join(b) for b in blocks).strip()
|
tinybird/tb/modules/tag.py
CHANGED
|
@@ -3,12 +3,12 @@ from typing import Optional
|
|
|
3
3
|
import click
|
|
4
4
|
from click import Context
|
|
5
5
|
|
|
6
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
7
6
|
from tinybird.tb.modules.cli import cli
|
|
8
7
|
from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table
|
|
8
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
@cli.group()
|
|
11
|
+
@cli.group(hidden=True)
|
|
12
12
|
@click.pass_context
|
|
13
13
|
def tag(ctx: Context) -> None:
|
|
14
14
|
"""Tag commands"""
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -7,20 +7,20 @@ import difflib
|
|
|
7
7
|
import glob
|
|
8
8
|
import os
|
|
9
9
|
from pathlib import Path
|
|
10
|
-
from typing import Iterable, List, Optional, Tuple
|
|
10
|
+
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
11
11
|
|
|
12
12
|
import click
|
|
13
13
|
import yaml
|
|
14
14
|
|
|
15
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
16
15
|
from tinybird.tb.modules.cli import cli
|
|
17
16
|
from tinybird.tb.modules.common import coro
|
|
18
17
|
from tinybird.tb.modules.config import CLIConfig
|
|
19
18
|
from tinybird.tb.modules.exceptions import CLIException
|
|
20
|
-
from tinybird.tb.modules.
|
|
19
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
20
|
+
from tinybird.tb.modules.llm import LLM
|
|
21
21
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
22
22
|
|
|
23
|
-
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str
|
|
23
|
+
yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def repr_str(dumper, data):
|
|
@@ -32,7 +32,7 @@ def repr_str(dumper, data):
|
|
|
32
32
|
yaml.add_representer(str, repr_str, Dumper=yaml.SafeDumper)
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def generate_test_file(pipe_name: str, tests: List[
|
|
35
|
+
def generate_test_file(pipe_name: str, tests: List[Dict[str, Any]], folder: Optional[str], mode: str = "w"):
|
|
36
36
|
base = Path("tests")
|
|
37
37
|
if folder:
|
|
38
38
|
base = Path(folder) / base
|
|
@@ -65,9 +65,8 @@ def test(ctx: click.Context) -> None:
|
|
|
65
65
|
help="Folder where datafiles will be placed",
|
|
66
66
|
)
|
|
67
67
|
@click.option("--prompt", type=str, default=None, help="Prompt to be used to create the test")
|
|
68
|
-
@click.pass_context
|
|
69
68
|
@coro
|
|
70
|
-
async def test_create(
|
|
69
|
+
async def test_create(pipe: str, prompt: Optional[str], folder: str) -> None:
|
|
71
70
|
"""
|
|
72
71
|
Create a test for an existing endpoint
|
|
73
72
|
"""
|
|
@@ -102,13 +101,21 @@ async def test_create(ctx: click.Context, pipe: str, prompt: Optional[str], fold
|
|
|
102
101
|
test_params = (
|
|
103
102
|
valid_test["parameters"] if valid_test["parameters"].startswith("?") else f"?{valid_test['parameters']}"
|
|
104
103
|
)
|
|
105
|
-
|
|
104
|
+
|
|
105
|
+
response = None
|
|
106
106
|
try:
|
|
107
|
-
response = await client.
|
|
107
|
+
response = await client._req_raw(f"/v0/pipes/{pipe_name}.ndjson{test_params}")
|
|
108
108
|
except Exception:
|
|
109
|
-
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
if response.status_code >= 400:
|
|
112
|
+
valid_test["expected_http_status"] = response.status_code
|
|
113
|
+
valid_test["expected_result"] = response.json()["error"]
|
|
114
|
+
else:
|
|
115
|
+
if "expected_http_status" in valid_test:
|
|
116
|
+
del valid_test["expected_http_status"]
|
|
117
|
+
valid_test["expected_result"] = response.text or ""
|
|
110
118
|
|
|
111
|
-
valid_test["expected_result"] = response
|
|
112
119
|
valid_test_expectations.append(valid_test)
|
|
113
120
|
if valid_test_expectations:
|
|
114
121
|
generate_test_file(pipe_name, valid_test_expectations, folder, mode="a")
|
|
@@ -129,11 +136,9 @@ async def test_create(ctx: click.Context, pipe: str, prompt: Optional[str], fold
|
|
|
129
136
|
type=click.Path(exists=True, file_okay=False),
|
|
130
137
|
help="Folder where datafiles will be placed",
|
|
131
138
|
)
|
|
132
|
-
@click.pass_context
|
|
133
139
|
@coro
|
|
134
|
-
async def test_update(
|
|
140
|
+
async def test_update(pipe: str, folder: str) -> None:
|
|
135
141
|
client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
136
|
-
|
|
137
142
|
pipe_tests_path = Path(pipe)
|
|
138
143
|
pipe_name = pipe
|
|
139
144
|
if pipe_tests_path.suffix == ".yaml":
|
|
@@ -146,16 +151,20 @@ async def test_update(ctx: click.Context, pipe: str, folder: Optional[str]) -> N
|
|
|
146
151
|
pipe_tests_content = yaml.safe_load(pipe_tests_path.read_text())
|
|
147
152
|
for test in pipe_tests_content:
|
|
148
153
|
test_params = test["parameters"] if test["parameters"].startswith("?") else f"?{test['parameters']}"
|
|
149
|
-
response =
|
|
154
|
+
response = None
|
|
150
155
|
try:
|
|
151
|
-
response = await client.
|
|
156
|
+
response = await client._req_raw(f"/v0/pipes/{pipe_name}.ndjson{test_params}")
|
|
152
157
|
except Exception:
|
|
153
|
-
|
|
158
|
+
continue
|
|
159
|
+
|
|
160
|
+
if response.status_code >= 400:
|
|
161
|
+
test["expected_http_status"] = response.status_code
|
|
162
|
+
test["expected_result"] = response.json()["error"]
|
|
154
163
|
else:
|
|
155
164
|
if "expected_http_status" in test:
|
|
156
165
|
del test["expected_http_status"]
|
|
157
166
|
|
|
158
|
-
|
|
167
|
+
test["expected_result"] = response.text or ""
|
|
159
168
|
|
|
160
169
|
generate_test_file(pipe_name, pipe_tests_content, folder)
|
|
161
170
|
click.echo(FeedbackManager.info(message=f"✓ /tests/{pipe_name}.yaml"))
|
|
@@ -166,18 +175,19 @@ async def test_update(ctx: click.Context, pipe: str, folder: Optional[str]) -> N
|
|
|
166
175
|
name="run",
|
|
167
176
|
help="Run the test suite, a file, or a test.",
|
|
168
177
|
)
|
|
169
|
-
@click.argument("
|
|
178
|
+
@click.argument("name", nargs=-1)
|
|
170
179
|
@click.option(
|
|
171
180
|
"--folder",
|
|
172
181
|
default=".",
|
|
173
182
|
type=click.Path(exists=True, file_okay=False),
|
|
174
183
|
help="Folder where tests will be placed",
|
|
175
184
|
)
|
|
176
|
-
@click.pass_context
|
|
177
185
|
@coro
|
|
178
|
-
async def test_run(
|
|
186
|
+
async def test_run(name: Tuple[str, ...], folder: str) -> None:
|
|
179
187
|
client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
180
|
-
|
|
188
|
+
paths = [Path(n) for n in name]
|
|
189
|
+
endpoints = [f"./tests/{p.stem}.yaml" for p in paths]
|
|
190
|
+
file_list: Iterable[str] = endpoints if len(endpoints) > 0 else glob.glob("./tests/**/*.y*ml", recursive=True)
|
|
181
191
|
|
|
182
192
|
async def run_test(test_file):
|
|
183
193
|
test_file_path = Path(test_file)
|
|
@@ -185,22 +195,28 @@ async def test_run(ctx: click.Context, file: Tuple[str, ...], folder: Optional[s
|
|
|
185
195
|
for test in test_file_content:
|
|
186
196
|
try:
|
|
187
197
|
test_params = test["parameters"] if test["parameters"].startswith("?") else f"?{test['parameters']}"
|
|
188
|
-
|
|
198
|
+
|
|
199
|
+
response = None
|
|
189
200
|
try:
|
|
190
|
-
response = await client.
|
|
201
|
+
response = await client._req_raw(f"/v0/pipes/{test_file_path.stem}.ndjson{test_params}")
|
|
191
202
|
except Exception:
|
|
203
|
+
raise Exception("Expected to not fail but got an error")
|
|
204
|
+
|
|
205
|
+
expected_result = response.text
|
|
206
|
+
if response.status_code >= 400:
|
|
207
|
+
expected_result = response.json()["error"]
|
|
192
208
|
if "expected_http_status" not in test:
|
|
193
209
|
raise Exception("Expected to not fail but got an error")
|
|
194
|
-
if test["expected_http_status"] !=
|
|
195
|
-
raise Exception(f"Expected {test['expected_http_status']} but got
|
|
210
|
+
if test["expected_http_status"] != response.status_code:
|
|
211
|
+
raise Exception(f"Expected {test['expected_http_status']} but got {response.status_code}")
|
|
196
212
|
|
|
197
|
-
if test["expected_result"] !=
|
|
213
|
+
if test["expected_result"] != expected_result:
|
|
198
214
|
diff = difflib.ndiff(
|
|
199
|
-
test["expected_result"].splitlines(keepends=True),
|
|
215
|
+
test["expected_result"].splitlines(keepends=True), expected_result.splitlines(keepends=True)
|
|
200
216
|
)
|
|
201
217
|
printable_diff = "".join(diff)
|
|
202
218
|
raise Exception(
|
|
203
|
-
f"\nExpected: \n{test['expected_result']}\nGot: \n{
|
|
219
|
+
f"\nExpected: \n{test['expected_result']}\nGot: \n{expected_result}\nDiff: \n{printable_diff}"
|
|
204
220
|
)
|
|
205
221
|
click.echo(FeedbackManager.success(message=f"✓ {test_file_path.name} - {test['name']}"))
|
|
206
222
|
except Exception as e:
|
|
@@ -8,8 +8,8 @@ from humanfriendly.tables import format_smart_table
|
|
|
8
8
|
from typing_extensions import override
|
|
9
9
|
|
|
10
10
|
from tinybird.client import TinyB
|
|
11
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
12
11
|
from tinybird.tb.modules.common import CLIException
|
|
12
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
@dataclass
|
tinybird/tb/modules/token.py
CHANGED
|
@@ -7,7 +7,6 @@ from click import Context
|
|
|
7
7
|
from humanfriendly import parse_timespan
|
|
8
8
|
|
|
9
9
|
from tinybird.client import AuthNoTokenException, TinyB
|
|
10
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
11
10
|
from tinybird.tb.modules.cli import cli
|
|
12
11
|
from tinybird.tb.modules.common import (
|
|
13
12
|
DoesNotExistException,
|
|
@@ -15,9 +14,10 @@ from tinybird.tb.modules.common import (
|
|
|
15
14
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
16
15
|
)
|
|
17
16
|
from tinybird.tb.modules.exceptions import CLITokenException
|
|
17
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
@cli.group()
|
|
20
|
+
@cli.group(hidden=True)
|
|
21
21
|
@click.pass_context
|
|
22
22
|
def token(ctx: Context) -> None:
|
|
23
23
|
"""Token commands."""
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import time
|
|
3
|
+
from typing import Any, Callable, List
|
|
4
|
+
|
|
5
|
+
import click
|
|
6
|
+
from watchdog.events import FileSystemEventHandler
|
|
7
|
+
from watchdog.observers import Observer
|
|
8
|
+
|
|
9
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
10
|
+
from tinybird.tb.modules.shell import Shell
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FileChangeHandler(FileSystemEventHandler):
|
|
14
|
+
def __init__(self, filenames: List[str], process: Callable[[List[str]], None], build_ok: bool):
|
|
15
|
+
self.filenames = filenames
|
|
16
|
+
self.process = process
|
|
17
|
+
self.build_ok = build_ok
|
|
18
|
+
|
|
19
|
+
def on_modified(self, event: Any) -> None:
|
|
20
|
+
is_not_vendor = "vendor/" not in event.src_path
|
|
21
|
+
if (
|
|
22
|
+
is_not_vendor
|
|
23
|
+
and not event.is_directory
|
|
24
|
+
and any(event.src_path.endswith(ext) for ext in [".datasource", ".pipe", ".ndjson"])
|
|
25
|
+
):
|
|
26
|
+
filename = event.src_path.split("/")[-1]
|
|
27
|
+
click.echo(FeedbackManager.highlight(message=f"\n\n⟲ Changes detected in {filename}\n"))
|
|
28
|
+
try:
|
|
29
|
+
to_process = [event.src_path] if self.build_ok else self.filenames
|
|
30
|
+
self.process(to_process)
|
|
31
|
+
self.build_ok = True
|
|
32
|
+
except Exception as e:
|
|
33
|
+
click.echo(FeedbackManager.error_exception(error=e))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def watch_files(
|
|
37
|
+
filenames: List[str],
|
|
38
|
+
process: Callable,
|
|
39
|
+
shell: Shell,
|
|
40
|
+
folder: str,
|
|
41
|
+
build_ok: bool,
|
|
42
|
+
) -> None:
|
|
43
|
+
# Handle both sync and async process functions
|
|
44
|
+
async def process_wrapper(files: List[str]) -> None:
|
|
45
|
+
click.echo("⚡ Rebuilding...")
|
|
46
|
+
time_start = time.time()
|
|
47
|
+
if asyncio.iscoroutinefunction(process):
|
|
48
|
+
await process(files, watch=True)
|
|
49
|
+
else:
|
|
50
|
+
process(files, watch=True)
|
|
51
|
+
time_end = time.time()
|
|
52
|
+
elapsed_time = time_end - time_start
|
|
53
|
+
click.echo(
|
|
54
|
+
FeedbackManager.success(message="\n✓ ")
|
|
55
|
+
+ FeedbackManager.gray(message=f"Rebuild completed in {elapsed_time:.1f}s")
|
|
56
|
+
)
|
|
57
|
+
shell.reprint_prompt()
|
|
58
|
+
|
|
59
|
+
event_handler = FileChangeHandler(filenames, lambda f: asyncio.run(process_wrapper(f)), build_ok)
|
|
60
|
+
observer = Observer()
|
|
61
|
+
|
|
62
|
+
observer.schedule(event_handler, path=folder, recursive=True)
|
|
63
|
+
|
|
64
|
+
observer.start()
|
|
65
|
+
|
|
66
|
+
try:
|
|
67
|
+
while True:
|
|
68
|
+
time.sleep(1)
|
|
69
|
+
except KeyboardInterrupt:
|
|
70
|
+
observer.stop()
|
|
71
|
+
|
|
72
|
+
observer.join()
|
tinybird/tb/modules/workspace.py
CHANGED
|
@@ -10,7 +10,6 @@ from click import Context
|
|
|
10
10
|
|
|
11
11
|
from tinybird.client import CanNotBeDeletedException, DoesNotExistException, TinyB
|
|
12
12
|
from tinybird.config import get_display_host
|
|
13
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
14
13
|
from tinybird.tb.modules.cli import cli
|
|
15
14
|
from tinybird.tb.modules.common import (
|
|
16
15
|
_get_workspace_plan_name,
|
|
@@ -28,6 +27,7 @@ from tinybird.tb.modules.common import (
|
|
|
28
27
|
from tinybird.tb.modules.config import CLIConfig
|
|
29
28
|
from tinybird.tb.modules.datafile.common import PipeTypes
|
|
30
29
|
from tinybird.tb.modules.exceptions import CLIWorkspaceException
|
|
30
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
@cli.group()
|
|
@@ -12,7 +12,6 @@ from click import Context
|
|
|
12
12
|
|
|
13
13
|
from tinybird.client import TinyB
|
|
14
14
|
from tinybird.config import get_display_host
|
|
15
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
16
15
|
from tinybird.tb.modules.common import (
|
|
17
16
|
ask_for_user_token,
|
|
18
17
|
check_user_token,
|
|
@@ -22,6 +21,7 @@ from tinybird.tb.modules.common import (
|
|
|
22
21
|
)
|
|
23
22
|
from tinybird.tb.modules.config import CLIConfig
|
|
24
23
|
from tinybird.tb.modules.exceptions import CLIWorkspaceMembersException
|
|
24
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
25
25
|
from tinybird.tb.modules.workspace import workspace
|
|
26
26
|
|
|
27
27
|
ROLES = ["viewer", "guest", "admin"]
|