tinybird 0.0.1.dev19__py3-none-any.whl → 0.0.1.dev20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/client.py +2 -2
- tinybird/config.py +1 -1
- tinybird/feedback_manager.py +8 -30
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/auth.py +1 -1
- tinybird/tb/modules/build.py +10 -78
- tinybird/tb/modules/cicd.py +1 -1
- tinybird/tb/modules/cli.py +9 -837
- tinybird/tb/modules/common.py +1 -55
- tinybird/tb/modules/connection.py +1 -1
- tinybird/tb/modules/create.py +18 -7
- tinybird/tb/modules/datafile/build.py +142 -971
- tinybird/tb/modules/datafile/build_common.py +1 -1
- tinybird/tb/modules/datafile/build_datasource.py +1 -1
- tinybird/tb/modules/datafile/build_pipe.py +1 -1
- tinybird/tb/modules/datafile/common.py +11 -11
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/fixture.py +1 -1
- tinybird/tb/modules/datafile/format_common.py +0 -7
- tinybird/tb/modules/datafile/format_datasource.py +0 -2
- tinybird/tb/modules/datafile/format_pipe.py +0 -2
- tinybird/tb/modules/datafile/parse_datasource.py +1 -1
- tinybird/tb/modules/datafile/parse_pipe.py +1 -1
- tinybird/tb/modules/datafile/pull.py +1 -1
- tinybird/tb/modules/datasource.py +4 -75
- tinybird/tb/modules/feedback_manager.py +1048 -0
- tinybird/tb/modules/fmt.py +1 -1
- tinybird/tb/modules/job.py +1 -1
- tinybird/tb/modules/llm.py +6 -4
- tinybird/tb/modules/local.py +26 -21
- tinybird/tb/modules/local_common.py +2 -1
- tinybird/tb/modules/login.py +8 -8
- tinybird/tb/modules/mock.py +18 -7
- tinybird/tb/modules/pipe.py +4 -126
- tinybird/tb/modules/{build_shell.py → shell.py} +9 -21
- tinybird/tb/modules/table.py +88 -5
- tinybird/tb/modules/tag.py +2 -2
- tinybird/tb/modules/test.py +8 -11
- tinybird/tb/modules/tinyunit/tinyunit.py +1 -1
- tinybird/tb/modules/token.py +2 -2
- tinybird/tb/modules/watch.py +72 -0
- tinybird/tb/modules/workspace.py +1 -1
- tinybird/tb/modules/workspace_members.py +1 -1
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev20.dist-info/RECORD +76 -0
- tinybird-0.0.1.dev19.dist-info/RECORD +0 -74
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev19.dist-info → tinybird-0.0.1.dev20.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/fmt.py
CHANGED
|
@@ -7,13 +7,13 @@ import aiofiles
|
|
|
7
7
|
import click
|
|
8
8
|
from click import Context
|
|
9
9
|
|
|
10
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
11
10
|
from tinybird.tb.modules.cli import cli
|
|
12
11
|
from tinybird.tb.modules.common import coro
|
|
13
12
|
from tinybird.tb.modules.datafile.common import is_file_a_datasource
|
|
14
13
|
from tinybird.tb.modules.datafile.diff import color_diff, peek
|
|
15
14
|
from tinybird.tb.modules.datafile.format_datasource import format_datasource
|
|
16
15
|
from tinybird.tb.modules.datafile.format_pipe import format_pipe
|
|
16
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
@cli.command()
|
tinybird/tb/modules/job.py
CHANGED
|
@@ -7,10 +7,10 @@ import click
|
|
|
7
7
|
from click import Context
|
|
8
8
|
|
|
9
9
|
from tinybird.client import DoesNotExistException, TinyB
|
|
10
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
11
10
|
from tinybird.tb.modules.cli import cli
|
|
12
11
|
from tinybird.tb.modules.common import coro, echo_safe_humanfriendly_tables_format_smart_table
|
|
13
12
|
from tinybird.tb.modules.exceptions import CLIException
|
|
13
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
@cli.group()
|
tinybird/tb/modules/llm.py
CHANGED
|
@@ -37,7 +37,8 @@ class LLM:
|
|
|
37
37
|
self.client = client
|
|
38
38
|
user_token = CLIConfig.get_project_config().get_user_token()
|
|
39
39
|
user_client = deepcopy(client)
|
|
40
|
-
|
|
40
|
+
if user_token:
|
|
41
|
+
user_client.token = user_token
|
|
41
42
|
self.user_client = user_client
|
|
42
43
|
self.openai = OpenAI(api_key=api_key) if api_key else None
|
|
43
44
|
|
|
@@ -75,10 +76,11 @@ class LLM:
|
|
|
75
76
|
data=f'{{"schema": "{urllib.parse.quote(schema)}", "rows": {rows}, "context": "{prompt}"}}',
|
|
76
77
|
headers={"Content-Type": "application/json"},
|
|
77
78
|
)
|
|
78
|
-
|
|
79
|
+
result = response.get("result", "")
|
|
80
|
+
return result.replace("elementAt", "arrayElement")
|
|
79
81
|
|
|
80
82
|
async def create_test_commands(
|
|
81
|
-
self, pipe_content: str, pipe_params: set[str], context: str =
|
|
83
|
+
self, pipe_content: str, pipe_params: set[str], context: Optional[str] = None
|
|
82
84
|
) -> TestExpectations:
|
|
83
85
|
if not self.openai:
|
|
84
86
|
raise ValueError("OpenAI API key is not set")
|
|
@@ -86,7 +88,7 @@ class LLM:
|
|
|
86
88
|
completion = self.openai.beta.chat.completions.parse(
|
|
87
89
|
model="gpt-4o",
|
|
88
90
|
messages=[
|
|
89
|
-
{"role": "system", "content": create_test_calls_prompt.format(context=context)},
|
|
91
|
+
{"role": "system", "content": create_test_calls_prompt.format(context=context or "")},
|
|
90
92
|
{"role": "user", "content": f"Pipe content: {pipe_content}\nPipe params: {pipe_params}"},
|
|
91
93
|
],
|
|
92
94
|
temperature=0.2,
|
tinybird/tb/modules/local.py
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import os
|
|
2
|
+
import re
|
|
2
3
|
import time
|
|
3
4
|
|
|
4
5
|
import click
|
|
5
6
|
|
|
6
7
|
import docker
|
|
7
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
8
8
|
from tinybird.tb.modules.cli import cli
|
|
9
9
|
from tinybird.tb.modules.common import coro
|
|
10
10
|
from tinybird.tb.modules.exceptions import CLIException
|
|
11
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
11
12
|
from tinybird.tb.modules.local_common import TB_CONTAINER_NAME, TB_IMAGE_NAME, TB_LOCAL_PORT
|
|
12
13
|
|
|
13
14
|
|
|
@@ -28,9 +29,9 @@ def start_tinybird_local(
|
|
|
28
29
|
|
|
29
30
|
if (
|
|
30
31
|
pull_show_prompt
|
|
31
|
-
and click.prompt(FeedbackManager.
|
|
32
|
+
and click.prompt(FeedbackManager.warning(message="△ New version detected, download? [y/N]")).lower() == "y"
|
|
32
33
|
):
|
|
33
|
-
click.echo(FeedbackManager.info(message="
|
|
34
|
+
click.echo(FeedbackManager.info(message="* Downloading latest version of Tinybird Local..."))
|
|
34
35
|
pull_required = True
|
|
35
36
|
|
|
36
37
|
if pull_required:
|
|
@@ -57,7 +58,7 @@ def start_tinybird_local(
|
|
|
57
58
|
platform="linux/amd64",
|
|
58
59
|
)
|
|
59
60
|
|
|
60
|
-
click.echo(FeedbackManager.info(message="
|
|
61
|
+
click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
|
|
61
62
|
for attempt in range(10):
|
|
62
63
|
try:
|
|
63
64
|
run = container.exec_run("tb --no-version-warning sql 'SELECT 1 AS healthcheck' --format json").output
|
|
@@ -68,16 +69,19 @@ def start_tinybird_local(
|
|
|
68
69
|
raise RuntimeError("Unexpected response from Tinybird")
|
|
69
70
|
except Exception:
|
|
70
71
|
if attempt == 9: # Last attempt
|
|
71
|
-
raise CLIException("Tinybird
|
|
72
|
+
raise CLIException("Tinybird Local not ready yet. Please try again in a few seconds.")
|
|
72
73
|
time.sleep(5) # Wait 5 seconds before retrying
|
|
73
74
|
|
|
74
|
-
|
|
75
|
+
# Remove tinybird-local dangling images to avoid running out of disk space
|
|
76
|
+
images = docker_client.images.list(name=re.sub(r":.*$", "", TB_IMAGE_NAME), all=True, filters={"dangling": True})
|
|
77
|
+
for image in images:
|
|
78
|
+
image.remove(force=True)
|
|
75
79
|
|
|
76
80
|
|
|
77
81
|
def get_docker_client():
|
|
78
82
|
"""Check if Docker is installed and running."""
|
|
79
83
|
try:
|
|
80
|
-
client = docker.from_env()
|
|
84
|
+
client = docker.from_env() # type: ignore
|
|
81
85
|
client.ping()
|
|
82
86
|
return client
|
|
83
87
|
except Exception:
|
|
@@ -105,9 +109,9 @@ def remove_tinybird_local(docker_client):
|
|
|
105
109
|
@cli.command()
|
|
106
110
|
def upgrade():
|
|
107
111
|
"""Upgrade Tinybird CLI to the latest version"""
|
|
108
|
-
click.echo(FeedbackManager.
|
|
112
|
+
click.echo(FeedbackManager.highlight(message="» Upgrading Tinybird CLI..."))
|
|
109
113
|
os.system(f"{os.getenv('HOME')}/.local/bin/uv tool upgrade tinybird")
|
|
110
|
-
click.echo(FeedbackManager.success(message="Tinybird CLI upgraded"))
|
|
114
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird CLI upgraded"))
|
|
111
115
|
|
|
112
116
|
|
|
113
117
|
@cli.group()
|
|
@@ -119,39 +123,40 @@ def local(ctx):
|
|
|
119
123
|
@local.command()
|
|
120
124
|
@coro
|
|
121
125
|
async def stop() -> None:
|
|
122
|
-
"""Stop Tinybird
|
|
123
|
-
click.echo(FeedbackManager.
|
|
126
|
+
"""Stop Tinybird Local"""
|
|
127
|
+
click.echo(FeedbackManager.highlight(message="» Shutting down Tinybird Local..."))
|
|
124
128
|
docker_client = get_docker_client()
|
|
125
129
|
stop_tinybird_local(docker_client)
|
|
126
|
-
click.echo(FeedbackManager.success(message="Tinybird
|
|
130
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local stopped"))
|
|
127
131
|
|
|
128
132
|
|
|
129
133
|
@local.command()
|
|
130
134
|
@coro
|
|
131
135
|
async def remove() -> None:
|
|
132
|
-
"""Remove Tinybird
|
|
133
|
-
click.echo(FeedbackManager.
|
|
136
|
+
"""Remove Tinybird Local"""
|
|
137
|
+
click.echo(FeedbackManager.highlight(message="» Removing Tinybird Local..."))
|
|
134
138
|
docker_client = get_docker_client()
|
|
135
139
|
remove_tinybird_local(docker_client)
|
|
136
|
-
click.echo(FeedbackManager.success(message="Tinybird
|
|
140
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local removed"))
|
|
137
141
|
|
|
138
142
|
|
|
139
143
|
@local.command()
|
|
140
144
|
@coro
|
|
141
145
|
async def start() -> None:
|
|
142
|
-
"""Start Tinybird
|
|
143
|
-
click.echo(FeedbackManager.
|
|
146
|
+
"""Start Tinybird Local"""
|
|
147
|
+
click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
|
|
144
148
|
docker_client = get_docker_client()
|
|
145
149
|
start_tinybird_local(docker_client)
|
|
146
|
-
click.echo(FeedbackManager.success(message="Tinybird
|
|
150
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
147
151
|
|
|
148
152
|
|
|
149
153
|
@local.command()
|
|
150
154
|
@coro
|
|
151
155
|
async def restart() -> None:
|
|
152
|
-
"""Restart Tinybird
|
|
153
|
-
click.echo(FeedbackManager.
|
|
156
|
+
"""Restart Tinybird Local"""
|
|
157
|
+
click.echo(FeedbackManager.highlight(message="» Restarting Tinybird Local..."))
|
|
154
158
|
docker_client = get_docker_client()
|
|
155
159
|
remove_tinybird_local(docker_client)
|
|
160
|
+
click.echo(FeedbackManager.info(message="✓ Tinybird Local stopped"))
|
|
156
161
|
start_tinybird_local(docker_client)
|
|
157
|
-
click.echo(FeedbackManager.success(message="Tinybird
|
|
162
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
@@ -8,7 +8,6 @@ from tinybird.client import TinyB
|
|
|
8
8
|
from tinybird.tb.modules.config import CLIConfig
|
|
9
9
|
from tinybird.tb.modules.exceptions import CLIException
|
|
10
10
|
|
|
11
|
-
# TODO: Use the official Tinybird image once it's available 'tinybirdco/tinybird-local:latest'
|
|
12
11
|
TB_IMAGE_NAME = "tinybirdco/tinybird-local:latest"
|
|
13
12
|
TB_CONTAINER_NAME = "tinybird-local"
|
|
14
13
|
TB_LOCAL_PORT = int(os.getenv("TB_LOCAL_PORT", 80))
|
|
@@ -40,6 +39,8 @@ async def get_tinybird_local_client(path: Optional[str] = None) -> TinyB:
|
|
|
40
39
|
await user_client.create_workspace(ws_name, template=None)
|
|
41
40
|
user_workspaces = await user_client.user_workspaces()
|
|
42
41
|
ws = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == ws_name), None)
|
|
42
|
+
if not ws:
|
|
43
|
+
raise CLIException(f"Workspace {ws_name} not found after creation")
|
|
43
44
|
|
|
44
45
|
ws_token = ws["token"]
|
|
45
46
|
|
tinybird/tb/modules/login.py
CHANGED
|
@@ -9,9 +9,9 @@ from urllib.parse import urlencode
|
|
|
9
9
|
import click
|
|
10
10
|
import requests
|
|
11
11
|
|
|
12
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
13
12
|
from tinybird.tb.modules.cli import CLIConfig, cli
|
|
14
13
|
from tinybird.tb.modules.common import coro
|
|
14
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class AuthHandler(http.server.SimpleHTTPRequestHandler):
|
|
@@ -75,7 +75,7 @@ class AuthHandler(http.server.SimpleHTTPRequestHandler):
|
|
|
75
75
|
|
|
76
76
|
if "token" in query_params:
|
|
77
77
|
token = query_params["token"][0]
|
|
78
|
-
self.server.auth_callback(token)
|
|
78
|
+
self.server.auth_callback(token) # type: ignore
|
|
79
79
|
self.send_response(200)
|
|
80
80
|
self.end_headers()
|
|
81
81
|
else:
|
|
@@ -158,15 +158,15 @@ async def login(host: str, workspace: str):
|
|
|
158
158
|
|
|
159
159
|
data = response.json()
|
|
160
160
|
cli_config = CLIConfig.get_project_config()
|
|
161
|
-
cli_config.set_token(data
|
|
162
|
-
cli_config.set_token_for_host(data
|
|
163
|
-
cli_config.set_user_token(data
|
|
161
|
+
cli_config.set_token(data.get("workspace_token", ""))
|
|
162
|
+
cli_config.set_token_for_host(data.get("workspace_token", ""), host)
|
|
163
|
+
cli_config.set_user_token(data.get("user_token", ""))
|
|
164
164
|
cli_config.set_host(host)
|
|
165
165
|
|
|
166
|
-
|
|
166
|
+
ws = await cli_config.get_client().workspace_info()
|
|
167
167
|
for k in ("id", "name", "user_email", "user_id", "scope"):
|
|
168
|
-
if k in
|
|
169
|
-
cli_config[k] =
|
|
168
|
+
if k in ws:
|
|
169
|
+
cli_config[k] = ws[k]
|
|
170
170
|
|
|
171
171
|
cli_config.persist_to_file()
|
|
172
172
|
|
tinybird/tb/modules/mock.py
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
|
+
import logging
|
|
1
2
|
import os
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
import click
|
|
5
6
|
|
|
6
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
7
7
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.common import CLIException, coro
|
|
8
|
+
from tinybird.tb.modules.common import CLIException, check_user_token, coro
|
|
9
9
|
from tinybird.tb.modules.config import CLIConfig
|
|
10
10
|
from tinybird.tb.modules.datafile.fixture import build_fixture_name, persist_fixture
|
|
11
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
11
12
|
from tinybird.tb.modules.llm import LLM
|
|
12
13
|
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
13
14
|
|
|
@@ -17,8 +18,9 @@ from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
|
17
18
|
@click.option("--rows", type=int, default=10, help="Number of events to send")
|
|
18
19
|
@click.option("--prompt", type=str, default="", help="Extra context to use for data generation")
|
|
19
20
|
@click.option("--folder", type=str, default=".", help="Folder where datafiles will be placed")
|
|
21
|
+
@click.pass_context
|
|
20
22
|
@coro
|
|
21
|
-
async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
|
|
23
|
+
async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, folder: str) -> None:
|
|
22
24
|
"""Load sample data into a Data Source.
|
|
23
25
|
|
|
24
26
|
Args:
|
|
@@ -49,15 +51,24 @@ async def mock(datasource: str, rows: int, prompt: str, folder: str) -> None:
|
|
|
49
51
|
datasource_content = datasource_path.read_text()
|
|
50
52
|
config = CLIConfig.get_project_config()
|
|
51
53
|
user_client = config.get_client()
|
|
52
|
-
|
|
54
|
+
user_token = config.get_user_token()
|
|
55
|
+
|
|
56
|
+
try:
|
|
57
|
+
if not user_token:
|
|
58
|
+
raise CLIException("No user token found")
|
|
59
|
+
await check_user_token(ctx, token=user_token)
|
|
60
|
+
except Exception:
|
|
61
|
+
click.echo(FeedbackManager.error(message="This action requires authentication. Run 'tb login' first."))
|
|
62
|
+
return
|
|
63
|
+
|
|
53
64
|
llm = LLM(client=user_client)
|
|
54
65
|
tb_client = await get_tinybird_local_client(os.path.abspath(folder))
|
|
55
66
|
sql = await llm.generate_sql_sample_data(datasource_content, rows=rows, prompt=prompt)
|
|
56
|
-
if os.environ.get(
|
|
57
|
-
|
|
67
|
+
if os.environ.get("TB_DEBUG", "") != "":
|
|
68
|
+
logging.debug(sql)
|
|
58
69
|
result = await tb_client.query(f"{sql} FORMAT JSON")
|
|
59
70
|
data = result.get("data", [])[:rows]
|
|
60
|
-
fixture_name = build_fixture_name(datasource_path.absolute(), datasource_name, datasource_content)
|
|
71
|
+
fixture_name = build_fixture_name(datasource_path.absolute().as_posix(), datasource_name, datasource_content)
|
|
61
72
|
persist_fixture(fixture_name, data)
|
|
62
73
|
click.echo(FeedbackManager.success(message="✓ Done!"))
|
|
63
74
|
|
tinybird/tb/modules/pipe.py
CHANGED
|
@@ -13,10 +13,8 @@ import click
|
|
|
13
13
|
import humanfriendly
|
|
14
14
|
from click import Context
|
|
15
15
|
|
|
16
|
-
import tinybird.context as context
|
|
17
16
|
from tinybird.client import AuthNoTokenException, DoesNotExistException, TinyB
|
|
18
|
-
from tinybird.config import DEFAULT_API_HOST
|
|
19
|
-
from tinybird.feedback_manager import FeedbackManager
|
|
17
|
+
from tinybird.config import DEFAULT_API_HOST
|
|
20
18
|
from tinybird.tb.modules.cli import cli
|
|
21
19
|
from tinybird.tb.modules.common import (
|
|
22
20
|
coro,
|
|
@@ -24,12 +22,13 @@ from tinybird.tb.modules.common import (
|
|
|
24
22
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
25
23
|
wait_job,
|
|
26
24
|
)
|
|
27
|
-
from tinybird.tb.modules.datafile.build import
|
|
25
|
+
from tinybird.tb.modules.datafile.build import process_file
|
|
28
26
|
from tinybird.tb.modules.datafile.common import PipeNodeTypes, PipeTypes, get_name_version
|
|
29
27
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
28
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
30
29
|
|
|
31
30
|
|
|
32
|
-
@cli.group()
|
|
31
|
+
@cli.group(hidden=True)
|
|
33
32
|
@click.pass_context
|
|
34
33
|
def pipe(ctx):
|
|
35
34
|
"""Pipes commands"""
|
|
@@ -585,127 +584,6 @@ async def print_pipe(ctx: Context, pipe: str, query: str, format_: str):
|
|
|
585
584
|
click.echo(res)
|
|
586
585
|
|
|
587
586
|
|
|
588
|
-
@pipe.command(name="regression-test", short_help="Run regression tests using last requests")
|
|
589
|
-
@click.option(
|
|
590
|
-
"--debug",
|
|
591
|
-
is_flag=True,
|
|
592
|
-
default=False,
|
|
593
|
-
help="Prints internal representation, can be combined with any command to get more information.",
|
|
594
|
-
)
|
|
595
|
-
@click.option("--only-response-times", is_flag=True, default=False, help="Checks only response times")
|
|
596
|
-
@click.argument("filenames", type=click.Path(exists=True), nargs=-1, default=None)
|
|
597
|
-
@click.option("--workspace_map", nargs=2, type=str, multiple=True)
|
|
598
|
-
@click.option(
|
|
599
|
-
"--workspace",
|
|
600
|
-
nargs=2,
|
|
601
|
-
type=str,
|
|
602
|
-
multiple=True,
|
|
603
|
-
help="add a workspace path to the list of external workspaces, usage: --workspace name path/to/folder",
|
|
604
|
-
)
|
|
605
|
-
@click.option(
|
|
606
|
-
"--no-versions",
|
|
607
|
-
is_flag=True,
|
|
608
|
-
default=False,
|
|
609
|
-
help="when set, resource dependency versions are not used, it pushes the dependencies as-is",
|
|
610
|
-
)
|
|
611
|
-
@click.option(
|
|
612
|
-
"-l", "--limit", type=click.IntRange(0, 100), default=0, required=False, help="Number of requests to validate"
|
|
613
|
-
)
|
|
614
|
-
@click.option(
|
|
615
|
-
"--sample-by-params",
|
|
616
|
-
type=click.IntRange(1, 100),
|
|
617
|
-
default=1,
|
|
618
|
-
required=False,
|
|
619
|
-
help="When set, we will aggregate the pipe_stats_rt requests by extractURLParameterNames(assumeNotNull(url)) and for each combination we will take a sample of N requests",
|
|
620
|
-
)
|
|
621
|
-
@click.option(
|
|
622
|
-
"-m",
|
|
623
|
-
"--match",
|
|
624
|
-
multiple=True,
|
|
625
|
-
required=False,
|
|
626
|
-
help="Filter the checker requests by specific parameter. You can pass multiple parameters -m foo -m bar",
|
|
627
|
-
)
|
|
628
|
-
@click.option(
|
|
629
|
-
"-ff", "--failfast", is_flag=True, default=False, help="When set, the checker will exit as soon one test fails"
|
|
630
|
-
)
|
|
631
|
-
@click.option(
|
|
632
|
-
"--ignore-order", is_flag=True, default=False, help="When set, the checker will ignore the order of list properties"
|
|
633
|
-
)
|
|
634
|
-
@click.option(
|
|
635
|
-
"--validate-processed-bytes",
|
|
636
|
-
is_flag=True,
|
|
637
|
-
default=False,
|
|
638
|
-
help="When set, the checker will validate that the new version doesn't process more than 25% than the current version",
|
|
639
|
-
)
|
|
640
|
-
@click.option(
|
|
641
|
-
"--check-requests-from-main",
|
|
642
|
-
is_flag=True,
|
|
643
|
-
default=False,
|
|
644
|
-
help="When set, the checker will get Main Workspace requests",
|
|
645
|
-
hidden=True,
|
|
646
|
-
)
|
|
647
|
-
@click.option(
|
|
648
|
-
"--relative-change",
|
|
649
|
-
type=float,
|
|
650
|
-
default=0.01,
|
|
651
|
-
help="When set, the checker will validate the new version has less than this distance with the current version",
|
|
652
|
-
)
|
|
653
|
-
@click.pass_context
|
|
654
|
-
@coro
|
|
655
|
-
async def regression_test(
|
|
656
|
-
ctx: click.Context,
|
|
657
|
-
filenames: Optional[List[str]],
|
|
658
|
-
debug: bool,
|
|
659
|
-
only_response_times: bool,
|
|
660
|
-
workspace_map,
|
|
661
|
-
workspace: str,
|
|
662
|
-
no_versions: bool,
|
|
663
|
-
limit: int,
|
|
664
|
-
sample_by_params: int,
|
|
665
|
-
match: List[str],
|
|
666
|
-
failfast: bool,
|
|
667
|
-
ignore_order: bool,
|
|
668
|
-
validate_processed_bytes: bool,
|
|
669
|
-
check_requests_from_main: bool,
|
|
670
|
-
relative_change: float,
|
|
671
|
-
):
|
|
672
|
-
"""
|
|
673
|
-
Run regression tests on Tinybird
|
|
674
|
-
"""
|
|
675
|
-
|
|
676
|
-
ignore_sql_errors = FeatureFlags.ignore_sql_errors()
|
|
677
|
-
|
|
678
|
-
context.disable_template_security_validation.set(True)
|
|
679
|
-
await folder_push(
|
|
680
|
-
create_tb_client(ctx),
|
|
681
|
-
filenames,
|
|
682
|
-
dry_run=False,
|
|
683
|
-
check=True,
|
|
684
|
-
push_deps=False,
|
|
685
|
-
debug=debug,
|
|
686
|
-
force=False,
|
|
687
|
-
populate=False,
|
|
688
|
-
upload_fixtures=False,
|
|
689
|
-
wait=False,
|
|
690
|
-
ignore_sql_errors=ignore_sql_errors,
|
|
691
|
-
skip_confirmation=False,
|
|
692
|
-
only_response_times=only_response_times,
|
|
693
|
-
workspace_map=dict(workspace_map),
|
|
694
|
-
workspace_lib_paths=workspace,
|
|
695
|
-
no_versions=no_versions,
|
|
696
|
-
run_tests=True,
|
|
697
|
-
tests_to_run=limit,
|
|
698
|
-
tests_relative_change=relative_change,
|
|
699
|
-
tests_sample_by_params=sample_by_params,
|
|
700
|
-
tests_filter_by=match,
|
|
701
|
-
tests_failfast=failfast,
|
|
702
|
-
tests_ignore_order=ignore_order,
|
|
703
|
-
tests_validate_processed_bytes=validate_processed_bytes,
|
|
704
|
-
tests_check_requests_from_branch=check_requests_from_main,
|
|
705
|
-
)
|
|
706
|
-
return
|
|
707
|
-
|
|
708
|
-
|
|
709
587
|
@pipe_copy.command(name="run", short_help="Run an on-demand copy job")
|
|
710
588
|
@click.argument("pipe_name_or_id")
|
|
711
589
|
@click.option("--wait", is_flag=True, default=False, help="Wait for the copy job to finish")
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import concurrent.futures
|
|
3
3
|
import os
|
|
4
|
-
import random
|
|
5
4
|
import subprocess
|
|
6
5
|
import sys
|
|
7
6
|
from typing import List
|
|
@@ -16,8 +15,8 @@ from prompt_toolkit.shortcuts import CompleteStyle
|
|
|
16
15
|
from prompt_toolkit.styles import Style
|
|
17
16
|
|
|
18
17
|
from tinybird.client import TinyB
|
|
19
|
-
from tinybird.feedback_manager import FeedbackManager, bcolors
|
|
20
18
|
from tinybird.tb.modules.exceptions import CLIException
|
|
19
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager, bcolors
|
|
21
20
|
from tinybird.tb.modules.table import format_table
|
|
22
21
|
|
|
23
22
|
|
|
@@ -176,7 +175,7 @@ def _(event):
|
|
|
176
175
|
b.start_completion(select_first=False)
|
|
177
176
|
|
|
178
177
|
|
|
179
|
-
class
|
|
178
|
+
class Shell:
|
|
180
179
|
def __init__(
|
|
181
180
|
self,
|
|
182
181
|
folder: str,
|
|
@@ -195,8 +194,7 @@ class BuildShell:
|
|
|
195
194
|
self.endpoints = endpoints
|
|
196
195
|
self.prompt_message = "\ntb > "
|
|
197
196
|
self.commands = ["create", "mock", "test", "tb", "select"]
|
|
198
|
-
|
|
199
|
-
self.session = PromptSession(
|
|
197
|
+
self.session: PromptSession = PromptSession(
|
|
200
198
|
completer=DynamicCompleter(self.datasources, self.shared_datasources, self.endpoints, self.pipes),
|
|
201
199
|
complete_style=CompleteStyle.COLUMN,
|
|
202
200
|
complete_while_typing=True,
|
|
@@ -210,7 +208,7 @@ class BuildShell:
|
|
|
210
208
|
except Exception:
|
|
211
209
|
return None
|
|
212
210
|
|
|
213
|
-
def
|
|
211
|
+
def run(self):
|
|
214
212
|
while True:
|
|
215
213
|
try:
|
|
216
214
|
user_input = self.session.prompt(
|
|
@@ -328,9 +326,6 @@ class BuildShell:
|
|
|
328
326
|
|
|
329
327
|
|
|
330
328
|
def print_table_formatted(res: dict, name: str):
|
|
331
|
-
rebuild_colors = [bcolors.FAIL, bcolors.OKBLUE, bcolors.WARNING, bcolors.OKGREEN, bcolors.HEADER]
|
|
332
|
-
rebuild_index = random.randint(0, len(rebuild_colors) - 1)
|
|
333
|
-
rebuild_color = rebuild_colors[rebuild_index % len(rebuild_colors)]
|
|
334
329
|
data = []
|
|
335
330
|
limit = 5
|
|
336
331
|
for d in res["data"][:5]:
|
|
@@ -341,24 +336,17 @@ def print_table_formatted(res: dict, name: str):
|
|
|
341
336
|
elapsed = stats.get("elapsed", 0)
|
|
342
337
|
cols = len(meta)
|
|
343
338
|
try:
|
|
344
|
-
|
|
345
|
-
def print_message(message: str, color=bcolors.CGREY):
|
|
346
|
-
return f"{color}{message}{bcolors.ENDC}"
|
|
347
|
-
|
|
348
339
|
table = format_table(data, meta)
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
click.echo(
|
|
352
|
-
click.echo(colored_char)
|
|
353
|
-
click.echo(table_with_marker)
|
|
354
|
-
click.echo(colored_char)
|
|
340
|
+
click.echo(FeedbackManager.highlight(message=f"\n» Running {name}\n"))
|
|
341
|
+
click.echo(table)
|
|
342
|
+
click.echo("")
|
|
355
343
|
rows_read = humanfriendly.format_number(stats.get("rows_read", 0))
|
|
356
344
|
bytes_read = humanfriendly.format_size(stats.get("bytes_read", 0))
|
|
357
345
|
elapsed = humanfriendly.format_timespan(elapsed) if elapsed >= 1 else f"{elapsed * 1000:.2f}ms"
|
|
358
346
|
stats_message = f"» {bytes_read} ({rows_read} rows x {cols} cols) in {elapsed}"
|
|
359
347
|
rows_message = f"» Showing {limit} first rows" if row_count > limit else "» Showing all rows"
|
|
360
|
-
click.echo(
|
|
361
|
-
click.echo(
|
|
348
|
+
click.echo(FeedbackManager.success(message=stats_message))
|
|
349
|
+
click.echo(FeedbackManager.gray(message=rows_message))
|
|
362
350
|
except ValueError as exc:
|
|
363
351
|
if str(exc) == "max() arg is an empty sequence":
|
|
364
352
|
click.echo("------------")
|
tinybird/tb/modules/table.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# Standard library modules.
|
|
2
2
|
import collections
|
|
3
3
|
import re
|
|
4
|
+
from typing import Union
|
|
4
5
|
|
|
5
6
|
# Modules included in our package.
|
|
6
7
|
from humanfriendly.compat import coerce_string
|
|
7
|
-
from humanfriendly.tables import format_robust_table
|
|
8
8
|
from humanfriendly.terminal import (
|
|
9
9
|
ansi_strip,
|
|
10
10
|
ansi_width,
|
|
@@ -13,7 +13,7 @@ from humanfriendly.terminal import (
|
|
|
13
13
|
terminal_supports_colors,
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
-
from tinybird.feedback_manager import bcolors
|
|
16
|
+
from tinybird.tb.modules.feedback_manager import bcolors
|
|
17
17
|
|
|
18
18
|
NUMERIC_DATA_PATTERN = re.compile(r"^\d+(\.\d+)?$")
|
|
19
19
|
|
|
@@ -133,7 +133,7 @@ def format_pretty_table(data, column_names=None, column_types=None, horizontal_b
|
|
|
133
133
|
column_types = [highlight_column_type(t) for t in column_types]
|
|
134
134
|
data.insert(1, column_types)
|
|
135
135
|
# Calculate the maximum width of each column.
|
|
136
|
-
widths = collections.defaultdict(int)
|
|
136
|
+
widths: dict[int, int] = collections.defaultdict(int)
|
|
137
137
|
numeric_data = collections.defaultdict(list)
|
|
138
138
|
for row_index, row in enumerate(data):
|
|
139
139
|
for column_index, column in enumerate(row):
|
|
@@ -177,9 +177,92 @@ def highlight_column_name(name):
|
|
|
177
177
|
return ansi_wrap(name)
|
|
178
178
|
|
|
179
179
|
|
|
180
|
+
def highlight_column_name_robust(name):
|
|
181
|
+
return f"{bcolors.CGREY}{name}{bcolors.ENDC}"
|
|
182
|
+
|
|
183
|
+
|
|
180
184
|
def highlight_column_type(type):
|
|
181
185
|
return f"{bcolors.CGREY}\033[3m{type}\033[23m{bcolors.ENDC}"
|
|
182
186
|
|
|
183
187
|
|
|
184
|
-
def highlight_horizontal_bar(horizontal_bar):
|
|
185
|
-
return f"
|
|
188
|
+
def highlight_horizontal_bar(horizontal_bar: str) -> str:
|
|
189
|
+
return f"\033[38;5;247m\033[3m{horizontal_bar}\033[23m{bcolors.ENDC}"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def format_robust_table(data, column_names):
|
|
193
|
+
"""
|
|
194
|
+
Render tabular data with one column per line (allowing columns with line breaks).
|
|
195
|
+
|
|
196
|
+
:param data: An iterable (e.g. a :func:`tuple` or :class:`list`)
|
|
197
|
+
containing the rows of the table, where each row is an
|
|
198
|
+
iterable containing the columns of the table (strings).
|
|
199
|
+
:param column_names: An iterable of column names (strings).
|
|
200
|
+
:returns: The rendered table (a string).
|
|
201
|
+
|
|
202
|
+
Here's an example:
|
|
203
|
+
|
|
204
|
+
>>> from humanfriendly.tables import format_robust_table
|
|
205
|
+
>>> column_names = ['Version', 'Uploaded on', 'Downloads']
|
|
206
|
+
>>> humanfriendly_releases = [
|
|
207
|
+
... ['1.23', '2015-05-25', '218'],
|
|
208
|
+
... ['1.23.1', '2015-05-26', '1354'],
|
|
209
|
+
... ['1.24', '2015-05-26', '223'],
|
|
210
|
+
... ['1.25', '2015-05-26', '4319'],
|
|
211
|
+
... ['1.25.1', '2015-06-02', '197'],
|
|
212
|
+
... ]
|
|
213
|
+
>>> print(format_robust_table(humanfriendly_releases, column_names))
|
|
214
|
+
-----------------------
|
|
215
|
+
Version: 1.23
|
|
216
|
+
Uploaded on: 2015-05-25
|
|
217
|
+
Downloads: 218
|
|
218
|
+
-----------------------
|
|
219
|
+
Version: 1.23.1
|
|
220
|
+
Uploaded on: 2015-05-26
|
|
221
|
+
Downloads: 1354
|
|
222
|
+
-----------------------
|
|
223
|
+
Version: 1.24
|
|
224
|
+
Uploaded on: 2015-05-26
|
|
225
|
+
Downloads: 223
|
|
226
|
+
-----------------------
|
|
227
|
+
Version: 1.25
|
|
228
|
+
Uploaded on: 2015-05-26
|
|
229
|
+
Downloads: 4319
|
|
230
|
+
-----------------------
|
|
231
|
+
Version: 1.25.1
|
|
232
|
+
Uploaded on: 2015-06-02
|
|
233
|
+
Downloads: 197
|
|
234
|
+
-----------------------
|
|
235
|
+
|
|
236
|
+
The column names are highlighted in bold font and color so they stand out a
|
|
237
|
+
bit more (see :data:`.HIGHLIGHT_COLOR`).
|
|
238
|
+
"""
|
|
239
|
+
blocks: list[Union[str, list[str]]] = []
|
|
240
|
+
column_names = ["%s:" % n for n in normalize_columns(column_names)]
|
|
241
|
+
if terminal_supports_colors():
|
|
242
|
+
column_names = [highlight_column_name_robust(n) for n in column_names]
|
|
243
|
+
# Convert each row into one or more `name: value' lines (one per column)
|
|
244
|
+
# and group each `row of lines' into a block (i.e. rows become blocks).
|
|
245
|
+
for row in data:
|
|
246
|
+
lines = []
|
|
247
|
+
for column_index, column_text in enumerate(normalize_columns(row)):
|
|
248
|
+
stripped_column = column_text.strip()
|
|
249
|
+
if "\n" not in stripped_column:
|
|
250
|
+
# Columns without line breaks are formatted inline.
|
|
251
|
+
lines.append("%s %s" % (column_names[column_index], stripped_column))
|
|
252
|
+
else:
|
|
253
|
+
# Columns with line breaks could very well contain indented
|
|
254
|
+
# lines, so we'll put the column name on a separate line. This
|
|
255
|
+
# way any indentation remains intact, and it's easier to
|
|
256
|
+
# copy/paste the text.
|
|
257
|
+
lines.append(column_names[column_index])
|
|
258
|
+
lines.extend(column_text.rstrip().splitlines())
|
|
259
|
+
blocks.append(lines)
|
|
260
|
+
# Calculate the width of the row delimiter.
|
|
261
|
+
num_rows, num_columns = find_terminal_size()
|
|
262
|
+
longest_line = max(max(map(ansi_width, lines)) for lines in blocks)
|
|
263
|
+
delimiter = highlight_horizontal_bar("\n%s\n" % ("─" * min(longest_line, num_columns)))
|
|
264
|
+
# Force a delimiter at the start and end of the table.
|
|
265
|
+
blocks.insert(0, "")
|
|
266
|
+
blocks.append("")
|
|
267
|
+
# Embed the row delimiter between every two blocks.
|
|
268
|
+
return delimiter.join("\n".join(b) for b in blocks).strip()
|