tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev236__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +22 -18
- tinybird/tb/modules/local_common.py +13 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev236.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/top_level.txt +0 -0
|
@@ -7,7 +7,6 @@ from tinybird.datafile.common import PipeTypes, get_name_version
|
|
|
7
7
|
from tinybird.tb.client import TinyB
|
|
8
8
|
from tinybird.tb.modules.cli import cli
|
|
9
9
|
from tinybird.tb.modules.common import (
|
|
10
|
-
coro,
|
|
11
10
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
12
11
|
wait_job,
|
|
13
12
|
)
|
|
@@ -31,15 +30,14 @@ def materialization(ctx):
|
|
|
31
30
|
help="Force a type of the output",
|
|
32
31
|
)
|
|
33
32
|
@click.pass_context
|
|
34
|
-
|
|
35
|
-
async def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
33
|
+
def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
36
34
|
"""List materializations"""
|
|
37
35
|
|
|
38
36
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
39
|
-
pipes =
|
|
37
|
+
pipes = client.pipes(dependencies=True, node_attrs="name,materialized", attrs="name,updated_at,endpoint,type")
|
|
40
38
|
materializations = [p for p in pipes if p.get("type") == PipeTypes.MATERIALIZED]
|
|
41
39
|
materializations = sorted(materializations, key=lambda p: p["updated_at"])
|
|
42
|
-
datasources =
|
|
40
|
+
datasources = client.datasources()
|
|
43
41
|
columns = ["name", "updated at", "nodes", "target datasource"]
|
|
44
42
|
table_human_readable = []
|
|
45
43
|
table_machine_readable = []
|
|
@@ -90,8 +88,7 @@ async def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
|
90
88
|
help="Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
|
|
91
89
|
)
|
|
92
90
|
@click.pass_context
|
|
93
|
-
|
|
94
|
-
async def pipe_populate(
|
|
91
|
+
def pipe_populate(
|
|
95
92
|
ctx: click.Context,
|
|
96
93
|
pipe_name: str,
|
|
97
94
|
node: str,
|
|
@@ -108,7 +105,7 @@ async def pipe_populate(
|
|
|
108
105
|
|
|
109
106
|
cl: TinyB = ctx.ensure_object(dict)["client"]
|
|
110
107
|
|
|
111
|
-
pipe =
|
|
108
|
+
pipe = cl.pipe(pipe_name)
|
|
112
109
|
|
|
113
110
|
if pipe["type"] != PipeTypes.MATERIALIZED:
|
|
114
111
|
raise CLIPipeException(FeedbackManager.error_pipe_not_materialized(pipe=pipe_name))
|
|
@@ -124,7 +121,7 @@ async def pipe_populate(
|
|
|
124
121
|
|
|
125
122
|
node = materialized_ids[0]
|
|
126
123
|
|
|
127
|
-
response =
|
|
124
|
+
response = cl.populate_node(pipe_name, node, populate_condition=sql_condition, truncate=truncate)
|
|
128
125
|
if "job" not in response:
|
|
129
126
|
raise CLIPipeException(response)
|
|
130
127
|
|
|
@@ -135,4 +132,4 @@ async def pipe_populate(
|
|
|
135
132
|
else:
|
|
136
133
|
click.echo(FeedbackManager.info_populate_job_url(url=job_url))
|
|
137
134
|
if wait:
|
|
138
|
-
|
|
135
|
+
wait_job(cl, job_id, job_url, "Populating")
|
tinybird/tb/modules/mock.py
CHANGED
|
@@ -7,7 +7,7 @@ import click
|
|
|
7
7
|
from tinybird.prompts import mock_prompt
|
|
8
8
|
from tinybird.tb.client import TinyB
|
|
9
9
|
from tinybird.tb.modules.cli import cli
|
|
10
|
-
from tinybird.tb.modules.common import
|
|
10
|
+
from tinybird.tb.modules.common import push_data
|
|
11
11
|
from tinybird.tb.modules.config import CLIConfig
|
|
12
12
|
from tinybird.tb.modules.datafile.fixture import persist_fixture, persist_fixture_sql
|
|
13
13
|
from tinybird.tb.modules.exceptions import CLIMockException
|
|
@@ -34,8 +34,7 @@ from tinybird.tb.modules.project import Project
|
|
|
34
34
|
help="Format of the fixture to create",
|
|
35
35
|
)
|
|
36
36
|
@click.pass_context
|
|
37
|
-
|
|
38
|
-
async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
|
|
37
|
+
def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, format_: str) -> None:
|
|
39
38
|
"""Generate sample data for a data source.
|
|
40
39
|
|
|
41
40
|
Args:
|
|
@@ -71,7 +70,7 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
|
|
|
71
70
|
if not user_token:
|
|
72
71
|
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
73
72
|
|
|
74
|
-
data =
|
|
73
|
+
data = create_mock_data(
|
|
75
74
|
datasource_name,
|
|
76
75
|
datasource_content,
|
|
77
76
|
rows,
|
|
@@ -87,7 +86,7 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
|
|
|
87
86
|
fixture_path = persist_fixture(datasource_name, data, folder, format=format_)
|
|
88
87
|
click.echo(FeedbackManager.info(message=f"✓ /fixtures/{datasource_name}.{format_} created"))
|
|
89
88
|
if env == "cloud":
|
|
90
|
-
|
|
89
|
+
append_fixture(tb_client, datasource_name, str(fixture_path))
|
|
91
90
|
|
|
92
91
|
click.echo(FeedbackManager.success(message=f"✓ Sample data for {datasource_name} created with {rows} rows"))
|
|
93
92
|
|
|
@@ -95,12 +94,12 @@ async def mock(ctx: click.Context, datasource: str, rows: int, prompt: str, form
|
|
|
95
94
|
raise CLIMockException(FeedbackManager.error(message=str(e)))
|
|
96
95
|
|
|
97
96
|
|
|
98
|
-
|
|
97
|
+
def append_fixture(
|
|
99
98
|
tb_client: TinyB,
|
|
100
99
|
datasource_name: str,
|
|
101
100
|
url: str,
|
|
102
101
|
):
|
|
103
|
-
|
|
102
|
+
push_data(
|
|
104
103
|
tb_client,
|
|
105
104
|
datasource_name,
|
|
106
105
|
url,
|
|
@@ -110,7 +109,7 @@ async def append_fixture(
|
|
|
110
109
|
)
|
|
111
110
|
|
|
112
111
|
|
|
113
|
-
|
|
112
|
+
def create_mock_data(
|
|
114
113
|
datasource_name: str,
|
|
115
114
|
datasource_content: str,
|
|
116
115
|
rows: int,
|
|
@@ -136,7 +135,7 @@ async def create_mock_data(
|
|
|
136
135
|
sql = extract_xml(response, "sql")
|
|
137
136
|
sql_path = persist_fixture_sql(datasource_name, sql, folder)
|
|
138
137
|
sql_format = "JSON" if format_ == "ndjson" else "CSV"
|
|
139
|
-
result =
|
|
138
|
+
result = tb_client.query(f"SELECT * FROM ({sql}) LIMIT {rows} FORMAT {sql_format}")
|
|
140
139
|
if sql_format == "JSON":
|
|
141
140
|
data = result.get("data", [])[:rows]
|
|
142
141
|
error_response = result.get("error", None)
|
tinybird/tb/modules/open.py
CHANGED
|
@@ -5,7 +5,6 @@ from click import Context
|
|
|
5
5
|
|
|
6
6
|
from tinybird.tb.config import get_display_cloud_host
|
|
7
7
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.common import coro
|
|
9
8
|
from tinybird.tb.modules.exceptions import CLIException
|
|
10
9
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
11
10
|
from tinybird.tb.modules.local_common import get_build_workspace_name
|
|
@@ -17,8 +16,7 @@ from tinybird.tb.modules.local_common import get_build_workspace_name
|
|
|
17
16
|
help="Set the workspace you want to open. If unset, your current workspace will be used.",
|
|
18
17
|
)
|
|
19
18
|
@click.pass_context
|
|
20
|
-
|
|
21
|
-
async def open(ctx: Context, workspace: str):
|
|
19
|
+
def open(ctx: Context, workspace: str):
|
|
22
20
|
"""Open workspace in the browser."""
|
|
23
21
|
|
|
24
22
|
config = ctx.ensure_object(dict)["config"]
|
tinybird/tb/modules/pipe.py
CHANGED
|
@@ -13,7 +13,6 @@ from tinybird.datafile.common import get_name_version
|
|
|
13
13
|
from tinybird.tb.client import TinyB
|
|
14
14
|
from tinybird.tb.modules.cli import cli
|
|
15
15
|
from tinybird.tb.modules.common import (
|
|
16
|
-
coro,
|
|
17
16
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
18
17
|
)
|
|
19
18
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
@@ -36,12 +35,11 @@ def pipe(ctx):
|
|
|
36
35
|
help="Force a type of the output",
|
|
37
36
|
)
|
|
38
37
|
@click.pass_context
|
|
39
|
-
|
|
40
|
-
async def pipe_ls(ctx: Context, match: str, format_: str):
|
|
38
|
+
def pipe_ls(ctx: Context, match: str, format_: str):
|
|
41
39
|
"""List pipes"""
|
|
42
40
|
|
|
43
41
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
44
|
-
pipes =
|
|
42
|
+
pipes = client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
|
|
45
43
|
pipes = sorted(pipes, key=lambda p: p["updated_at"])
|
|
46
44
|
|
|
47
45
|
columns = ["name", "published date", "nodes", "type"]
|
tinybird/tb/modules/secret.py
CHANGED
|
@@ -4,10 +4,9 @@ from typing import Dict, Optional
|
|
|
4
4
|
import click
|
|
5
5
|
from dotenv import dotenv_values, set_key
|
|
6
6
|
|
|
7
|
-
from tinybird.syncasync import async_to_sync
|
|
8
7
|
from tinybird.tb.client import TinyB
|
|
9
8
|
from tinybird.tb.modules.cli import cli
|
|
10
|
-
from tinybird.tb.modules.common import
|
|
9
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
|
|
11
10
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
12
11
|
from tinybird.tb.modules.project import Project
|
|
13
12
|
|
|
@@ -21,12 +20,11 @@ def secret(ctx):
|
|
|
21
20
|
@secret.command(name="ls")
|
|
22
21
|
@click.option("--match", default=None, help="Retrieve any secrets matching the pattern. For example, --match _test")
|
|
23
22
|
@click.pass_context
|
|
24
|
-
|
|
25
|
-
async def secret_ls(ctx: click.Context, match: Optional[str]):
|
|
23
|
+
def secret_ls(ctx: click.Context, match: Optional[str]):
|
|
26
24
|
"""List secrets"""
|
|
27
25
|
|
|
28
26
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
29
|
-
secrets =
|
|
27
|
+
secrets = client.secrets()
|
|
30
28
|
columns = ["name", "created_at", "updated_at"]
|
|
31
29
|
table_human_readable = []
|
|
32
30
|
table_machine_readable = []
|
|
@@ -54,8 +52,7 @@ async def secret_ls(ctx: click.Context, match: Optional[str]):
|
|
|
54
52
|
@click.argument("value", required=False)
|
|
55
53
|
@click.option("--multiline", is_flag=True, help="Whether to use multiline input")
|
|
56
54
|
@click.pass_context
|
|
57
|
-
|
|
58
|
-
async def secret_set(ctx: click.Context, name: str, value: Optional[str], multiline: bool):
|
|
55
|
+
def secret_set(ctx: click.Context, name: str, value: Optional[str], multiline: bool):
|
|
59
56
|
"""Create or update secrets"""
|
|
60
57
|
try:
|
|
61
58
|
if not value:
|
|
@@ -73,14 +70,14 @@ async def secret_set(ctx: click.Context, name: str, value: Optional[str], multil
|
|
|
73
70
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
74
71
|
existing_secret = None
|
|
75
72
|
try:
|
|
76
|
-
existing_secret =
|
|
73
|
+
existing_secret = client.get_secret(name)
|
|
77
74
|
except Exception:
|
|
78
75
|
pass
|
|
79
76
|
|
|
80
77
|
if existing_secret:
|
|
81
|
-
|
|
78
|
+
client.update_secret(name, value)
|
|
82
79
|
else:
|
|
83
|
-
|
|
80
|
+
client.create_secret(name, value)
|
|
84
81
|
click.echo(FeedbackManager.success(message=f"\n✓ Secret '{name}' set"))
|
|
85
82
|
except Exception as e:
|
|
86
83
|
click.echo(FeedbackManager.error(message=f"✗ Error: {e}"))
|
|
@@ -89,13 +86,12 @@ async def secret_set(ctx: click.Context, name: str, value: Optional[str], multil
|
|
|
89
86
|
@secret.command(name="rm")
|
|
90
87
|
@click.argument("name")
|
|
91
88
|
@click.pass_context
|
|
92
|
-
|
|
93
|
-
async def secret_rm(ctx: click.Context, name: str):
|
|
89
|
+
def secret_rm(ctx: click.Context, name: str):
|
|
94
90
|
"""Delete a secret"""
|
|
95
91
|
try:
|
|
96
92
|
click.echo(FeedbackManager.highlight(message=f"\n» Deleting secret '{name}'..."))
|
|
97
93
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
98
|
-
|
|
94
|
+
client.delete_secret(name)
|
|
99
95
|
click.echo(FeedbackManager.success(message=f"\n✓ Secret '{name}' deleted"))
|
|
100
96
|
except Exception as e:
|
|
101
97
|
click.echo(FeedbackManager.error(message=f"✗ Error: {e}"))
|
|
@@ -133,14 +129,14 @@ def load_secrets(project: Project, client: TinyB):
|
|
|
133
129
|
continue
|
|
134
130
|
|
|
135
131
|
try:
|
|
136
|
-
existing_secret =
|
|
132
|
+
existing_secret = client.get_secret(name)
|
|
137
133
|
except Exception:
|
|
138
134
|
existing_secret = None
|
|
139
135
|
try:
|
|
140
136
|
if existing_secret:
|
|
141
|
-
|
|
137
|
+
client.update_secret(name, value)
|
|
142
138
|
else:
|
|
143
|
-
|
|
139
|
+
client.create_secret(name, value)
|
|
144
140
|
except Exception as e:
|
|
145
141
|
click.echo(FeedbackManager.error(message=f"✗ Error setting secret '{name}': {e}"))
|
|
146
142
|
|
tinybird/tb/modules/shell.py
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import concurrent.futures
|
|
3
1
|
import os
|
|
4
2
|
import subprocess
|
|
5
3
|
import sys
|
|
@@ -315,26 +313,15 @@ class Shell:
|
|
|
315
313
|
if q.lower().startswith("delete"):
|
|
316
314
|
raise CLIException(FeedbackManager.error_invalid_query())
|
|
317
315
|
|
|
318
|
-
|
|
319
|
-
loop = asyncio.new_event_loop()
|
|
320
|
-
asyncio.set_event_loop(loop)
|
|
321
|
-
try:
|
|
322
|
-
return loop.run_until_complete(
|
|
323
|
-
self.tb_client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON")
|
|
324
|
-
)
|
|
325
|
-
finally:
|
|
326
|
-
loop.close()
|
|
327
|
-
|
|
328
|
-
with concurrent.futures.ThreadPoolExecutor() as executor:
|
|
329
|
-
res = executor.submit(run_query_in_thread).result()
|
|
316
|
+
res = self.tb_client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON")
|
|
330
317
|
|
|
331
|
-
|
|
332
|
-
|
|
318
|
+
if isinstance(res, dict) and "error" in res:
|
|
319
|
+
click.echo(FeedbackManager.error_exception(error=res["error"]))
|
|
333
320
|
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
321
|
+
if isinstance(res, dict) and "data" in res and res["data"]:
|
|
322
|
+
print_table_formatted(res, "QUERY")
|
|
323
|
+
else:
|
|
324
|
+
click.echo(FeedbackManager.info_no_rows())
|
|
338
325
|
|
|
339
326
|
except Exception as e:
|
|
340
327
|
click.echo(FeedbackManager.error_exception(error=str(e)))
|
tinybird/tb/modules/sink.py
CHANGED
|
@@ -8,7 +8,7 @@ from click import Context
|
|
|
8
8
|
from tinybird.datafile.common import get_name_version
|
|
9
9
|
from tinybird.tb.client import AuthNoTokenException, TinyB
|
|
10
10
|
from tinybird.tb.modules.cli import cli
|
|
11
|
-
from tinybird.tb.modules.common import
|
|
11
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table, wait_job
|
|
12
12
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
13
13
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
14
14
|
|
|
@@ -29,12 +29,11 @@ def sink(ctx):
|
|
|
29
29
|
help="Force a type of the output",
|
|
30
30
|
)
|
|
31
31
|
@click.pass_context
|
|
32
|
-
|
|
33
|
-
async def sink_ls(ctx: Context, match: str, format_: str):
|
|
32
|
+
def sink_ls(ctx: Context, match: str, format_: str):
|
|
34
33
|
"""List sink pipes"""
|
|
35
34
|
|
|
36
35
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
|
-
pipes =
|
|
36
|
+
pipes = client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,type")
|
|
38
37
|
sinks = [p for p in pipes if p.get("type") == "sink"]
|
|
39
38
|
sinks = sorted(sinks, key=lambda p: p["updated_at"])
|
|
40
39
|
columns = ["name", "updated at", "nodes"]
|
|
@@ -79,8 +78,7 @@ async def sink_ls(ctx: Context, match: str, format_: str):
|
|
|
79
78
|
help="Key and value of the params you want the Sink pipe to be called with. For example: tb pipe sink run <my_sink_pipe> --param foo=bar",
|
|
80
79
|
)
|
|
81
80
|
@click.pass_context
|
|
82
|
-
|
|
83
|
-
async def sink_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, param: Optional[Tuple[str]]):
|
|
81
|
+
def sink_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: str, param: Optional[Tuple[str]]):
|
|
84
82
|
"""Run a sink pipe"""
|
|
85
83
|
|
|
86
84
|
params = dict(key_value.split("=") for key_value in param) if param else {}
|
|
@@ -88,7 +86,7 @@ async def sink_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: s
|
|
|
88
86
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
89
87
|
|
|
90
88
|
try:
|
|
91
|
-
response =
|
|
89
|
+
response = client.pipe_run(pipe_name_or_id, "sink", params, mode)
|
|
92
90
|
job_id = response["job"]["id"]
|
|
93
91
|
job_url = response["job"]["job_url"]
|
|
94
92
|
click.echo(FeedbackManager.gray(message="Job URL: ") + FeedbackManager.info(message=f"{job_url}"))
|
|
@@ -96,7 +94,7 @@ async def sink_run(ctx: click.Context, pipe_name_or_id: str, wait: bool, mode: s
|
|
|
96
94
|
|
|
97
95
|
if wait:
|
|
98
96
|
click.echo("\n")
|
|
99
|
-
|
|
97
|
+
wait_job(client, job_id, job_url, FeedbackManager.highlight(message="» Exporting data"))
|
|
100
98
|
click.echo(FeedbackManager.success(message="✓ Data exported"))
|
|
101
99
|
|
|
102
100
|
except AuthNoTokenException:
|
tinybird/tb/modules/test.py
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import asyncio
|
|
7
6
|
import difflib
|
|
8
7
|
import glob
|
|
9
8
|
import sys
|
|
@@ -89,7 +88,7 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
89
88
|
pipe_name = pipe_path.stem
|
|
90
89
|
click.echo(FeedbackManager.highlight(message=f"» Creating tests for {pipe_name} endpoint..."))
|
|
91
90
|
pipe_content = pipe_path.read_text()
|
|
92
|
-
pipe =
|
|
91
|
+
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
93
92
|
parameters = set([param["name"] for node in pipe["nodes"] for param in node["params"]])
|
|
94
93
|
|
|
95
94
|
system_prompt = test_create_prompt.format(
|
|
@@ -118,7 +117,7 @@ def test_create(ctx: click.Context, name_or_filename: str, prompt: str) -> None:
|
|
|
118
117
|
|
|
119
118
|
response = None
|
|
120
119
|
try:
|
|
121
|
-
response =
|
|
120
|
+
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test["parameters"])
|
|
122
121
|
except Exception:
|
|
123
122
|
pass
|
|
124
123
|
|
|
@@ -176,7 +175,7 @@ def test_update(ctx: click.Context, pipe: str) -> None:
|
|
|
176
175
|
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
177
176
|
response = None
|
|
178
177
|
try:
|
|
179
|
-
response =
|
|
178
|
+
response = get_pipe_data(client, pipe_name=pipe_name, test_params=test_params)
|
|
180
179
|
except Exception:
|
|
181
180
|
continue
|
|
182
181
|
|
|
@@ -233,9 +232,7 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
|
|
|
233
232
|
test_params = test["parameters"].split("?")[1] if "?" in test["parameters"] else test["parameters"]
|
|
234
233
|
response = None
|
|
235
234
|
try:
|
|
236
|
-
response =
|
|
237
|
-
get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
238
|
-
)
|
|
235
|
+
response = get_pipe_data(client, pipe_name=test_file_path.stem, test_params=test_params)
|
|
239
236
|
except Exception:
|
|
240
237
|
continue
|
|
241
238
|
|
|
@@ -281,20 +278,20 @@ def run_tests(ctx: click.Context, name: Tuple[str, ...]) -> None:
|
|
|
281
278
|
cleanup_test_workspace(client, project.folder)
|
|
282
279
|
|
|
283
280
|
|
|
284
|
-
|
|
285
|
-
pipe =
|
|
281
|
+
def get_pipe_data(client: TinyB, pipe_name: str, test_params: str) -> Response:
|
|
282
|
+
pipe = client._req(f"/v0/pipes/{pipe_name}")
|
|
286
283
|
output_node = next(
|
|
287
284
|
(node for node in pipe["nodes"] if node["node_type"] != "default" and node["node_type"] != "standard"),
|
|
288
285
|
{"name": "not_found"},
|
|
289
286
|
)
|
|
290
287
|
if output_node["node_type"] == "endpoint":
|
|
291
|
-
return
|
|
288
|
+
return client._req_raw(f"/v0/pipes/{pipe_name}.ndjson?{test_params}")
|
|
292
289
|
|
|
293
290
|
params = {
|
|
294
291
|
"q": output_node["sql"],
|
|
295
292
|
"pipeline": pipe_name,
|
|
296
293
|
}
|
|
297
|
-
return
|
|
294
|
+
return client._req_raw(f"""/v0/sql?{urllib.parse.urlencode(params)}&{test_params}""")
|
|
298
295
|
|
|
299
296
|
|
|
300
297
|
def get_pipe_path(name_or_filename: str, folder: str) -> Path:
|
|
@@ -320,8 +317,6 @@ def cleanup_test_workspace(client: TinyB, path: str) -> None:
|
|
|
320
317
|
try:
|
|
321
318
|
user_token = tokens["user_token"]
|
|
322
319
|
user_client.token = user_token
|
|
323
|
-
|
|
324
|
-
user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
|
|
325
|
-
)
|
|
320
|
+
user_client.delete_workspace(get_test_workspace_name(path), hard_delete_confirmation="yes", version="v1")
|
|
326
321
|
except Exception:
|
|
327
322
|
pass
|
|
@@ -191,7 +191,7 @@ def generate_file(file: str, overwrite: bool = False) -> None:
|
|
|
191
191
|
raise CLIException(FeedbackManager.error_file_already_exists(file=p))
|
|
192
192
|
|
|
193
193
|
|
|
194
|
-
|
|
194
|
+
def run_test_file(tb_client: TinyB, file: str) -> List[TestResult]:
|
|
195
195
|
results: List[TestResult] = []
|
|
196
196
|
for test_case in parse_file(file):
|
|
197
197
|
if not test_case.sql and not test_case.pipe:
|
|
@@ -211,7 +211,7 @@ async def run_test_file(tb_client: TinyB, file: str) -> List[TestResult]:
|
|
|
211
211
|
if test_case.sql and not test_case.pipe:
|
|
212
212
|
q = f"SELECT * FROM ({test_case.sql}) LIMIT 20 FORMAT JSON"
|
|
213
213
|
try:
|
|
214
|
-
test_response =
|
|
214
|
+
test_response = tb_client.query(q)
|
|
215
215
|
results.append(
|
|
216
216
|
TestResult(
|
|
217
217
|
name=test_case.name,
|
|
@@ -241,7 +241,7 @@ async def run_test_file(tb_client: TinyB, file: str) -> List[TestResult]:
|
|
|
241
241
|
params = test_case.pipe.params
|
|
242
242
|
try:
|
|
243
243
|
sql = test_case.sql if test_case.sql else None
|
|
244
|
-
test_response =
|
|
244
|
+
test_response = tb_client.pipe_data(pipe, format="json", params=params, sql=sql)
|
|
245
245
|
results.append(
|
|
246
246
|
TestResult(
|
|
247
247
|
name=test_case.name,
|
tinybird/tb/modules/token.py
CHANGED
|
@@ -10,7 +10,6 @@ from tinybird.tb.client import AuthNoTokenException, TinyB
|
|
|
10
10
|
from tinybird.tb.modules.cli import cli
|
|
11
11
|
from tinybird.tb.modules.common import (
|
|
12
12
|
DoesNotExistException,
|
|
13
|
-
coro,
|
|
14
13
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
15
14
|
)
|
|
16
15
|
from tinybird.tb.modules.exceptions import CLITokenException
|
|
@@ -26,8 +25,7 @@ def token(ctx: Context) -> None:
|
|
|
26
25
|
@token.command(name="ls")
|
|
27
26
|
@click.option("--match", default=None, help="Retrieve any token matching the pattern. For example, --match _test")
|
|
28
27
|
@click.pass_context
|
|
29
|
-
|
|
30
|
-
async def token_ls(
|
|
28
|
+
def token_ls(
|
|
31
29
|
ctx: Context,
|
|
32
30
|
match: Optional[str] = None,
|
|
33
31
|
) -> None:
|
|
@@ -37,7 +35,7 @@ async def token_ls(
|
|
|
37
35
|
client: TinyB = obj["client"]
|
|
38
36
|
|
|
39
37
|
try:
|
|
40
|
-
tokens =
|
|
38
|
+
tokens = client.token_list(match)
|
|
41
39
|
columns = ["id", "name", "token"]
|
|
42
40
|
table = list(map(lambda token: [token.get(key, "") for key in columns], tokens))
|
|
43
41
|
|
|
@@ -54,15 +52,14 @@ async def token_ls(
|
|
|
54
52
|
@click.argument("token_id")
|
|
55
53
|
@click.option("--yes", is_flag=True, default=False, help="Don't ask for confirmation")
|
|
56
54
|
@click.pass_context
|
|
57
|
-
|
|
58
|
-
async def token_rm(ctx: Context, token_id: str, yes: bool) -> None:
|
|
55
|
+
def token_rm(ctx: Context, token_id: str, yes: bool) -> None:
|
|
59
56
|
"""Remove a static token."""
|
|
60
57
|
|
|
61
58
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
62
59
|
client: TinyB = obj["client"]
|
|
63
60
|
if yes or click.confirm(FeedbackManager.warning_confirm_delete_token(token=token_id)):
|
|
64
61
|
try:
|
|
65
|
-
|
|
62
|
+
client.token_delete(token_id)
|
|
66
63
|
except AuthNoTokenException:
|
|
67
64
|
raise
|
|
68
65
|
except DoesNotExistException:
|
|
@@ -76,15 +73,14 @@ async def token_rm(ctx: Context, token_id: str, yes: bool) -> None:
|
|
|
76
73
|
@click.argument("token_id")
|
|
77
74
|
@click.option("--yes", is_flag=True, default=False, help="Don't ask for confirmation")
|
|
78
75
|
@click.pass_context
|
|
79
|
-
|
|
80
|
-
async def token_refresh(ctx: Context, token_id: str, yes: bool) -> None:
|
|
76
|
+
def token_refresh(ctx: Context, token_id: str, yes: bool) -> None:
|
|
81
77
|
"""Refresh a Static Token."""
|
|
82
78
|
|
|
83
79
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
84
80
|
client: TinyB = obj["client"]
|
|
85
81
|
if yes or click.confirm(FeedbackManager.warning_confirm_refresh_token(token=token_id)):
|
|
86
82
|
try:
|
|
87
|
-
|
|
83
|
+
client.token_refresh(token_id)
|
|
88
84
|
except AuthNoTokenException:
|
|
89
85
|
raise
|
|
90
86
|
except DoesNotExistException:
|
|
@@ -97,15 +93,14 @@ async def token_refresh(ctx: Context, token_id: str, yes: bool) -> None:
|
|
|
97
93
|
@token.command(name="scopes")
|
|
98
94
|
@click.argument("token_id")
|
|
99
95
|
@click.pass_context
|
|
100
|
-
|
|
101
|
-
async def token_scopes(ctx: Context, token_id: str) -> None:
|
|
96
|
+
def token_scopes(ctx: Context, token_id: str) -> None:
|
|
102
97
|
"""List Static Token scopes."""
|
|
103
98
|
|
|
104
99
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
105
100
|
client: TinyB = obj["client"]
|
|
106
101
|
|
|
107
102
|
try:
|
|
108
|
-
scopes =
|
|
103
|
+
scopes = client.token_scopes(token_id)
|
|
109
104
|
columns = ["type", "resource", "filter"]
|
|
110
105
|
table = list(map(lambda scope: [scope.get(key, "") for key in columns], scopes))
|
|
111
106
|
click.echo(FeedbackManager.info_token_scopes(token=token_id))
|
|
@@ -120,15 +115,14 @@ async def token_scopes(ctx: Context, token_id: str) -> None:
|
|
|
120
115
|
@token.command(name="copy")
|
|
121
116
|
@click.argument("token_id")
|
|
122
117
|
@click.pass_context
|
|
123
|
-
|
|
124
|
-
async def token_copy(ctx: Context, token_id: str) -> None:
|
|
118
|
+
def token_copy(ctx: Context, token_id: str) -> None:
|
|
125
119
|
"""Copy a Static Token."""
|
|
126
120
|
|
|
127
121
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
128
122
|
client: TinyB = obj["client"]
|
|
129
123
|
|
|
130
124
|
try:
|
|
131
|
-
token =
|
|
125
|
+
token = client.token_get(token_id)
|
|
132
126
|
pyperclip.copy(token["token"].strip())
|
|
133
127
|
except AuthNoTokenException:
|
|
134
128
|
raise
|
|
@@ -197,8 +191,7 @@ def create(ctx: Context) -> None:
|
|
|
197
191
|
"--fixed-params", multiple=True, help="Fixed parameters in key=value format, multiple values separated by commas"
|
|
198
192
|
)
|
|
199
193
|
@click.pass_context
|
|
200
|
-
|
|
201
|
-
async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resource, fixed_params) -> None:
|
|
194
|
+
def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resource, fixed_params) -> None:
|
|
202
195
|
"""Create a JWT token with a TTL specify."""
|
|
203
196
|
|
|
204
197
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
@@ -231,7 +224,7 @@ async def create_jwt_token(ctx: Context, name: str, ttl: timedelta, scope, resou
|
|
|
231
224
|
)
|
|
232
225
|
|
|
233
226
|
try:
|
|
234
|
-
response =
|
|
227
|
+
response = client.create_jwt_token(name, expiration_time, scopes)
|
|
235
228
|
except AuthNoTokenException:
|
|
236
229
|
raise
|
|
237
230
|
except Exception as e:
|
|
@@ -275,8 +268,7 @@ class DynamicOptionsCommand(click.Command):
|
|
|
275
268
|
)
|
|
276
269
|
@click.argument("name")
|
|
277
270
|
@click.pass_context
|
|
278
|
-
|
|
279
|
-
async def create_static_token(ctx, name: str):
|
|
271
|
+
def create_static_token(ctx, name: str):
|
|
280
272
|
"""Create a Static Token."""
|
|
281
273
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
282
274
|
client: TinyB = obj["client"]
|
|
@@ -336,15 +328,15 @@ async def create_static_token(ctx, name: str):
|
|
|
336
328
|
token = None
|
|
337
329
|
try:
|
|
338
330
|
click.echo(FeedbackManager.highlight(message=f"\n» Checking if token '{name}' exists..."))
|
|
339
|
-
token =
|
|
331
|
+
token = client.token_get(name)
|
|
340
332
|
except Exception:
|
|
341
333
|
pass
|
|
342
334
|
if token:
|
|
343
335
|
click.echo(FeedbackManager.info(message=f"* Token '{name}' found, updating it..."))
|
|
344
|
-
|
|
336
|
+
client.alter_tokens(name, scoped_parsed)
|
|
345
337
|
else:
|
|
346
338
|
click.echo(FeedbackManager.info(message=f"* Token '{name}' not found, creating it..."))
|
|
347
|
-
|
|
339
|
+
client.create_token(name, scoped_parsed, origin_code=None)
|
|
348
340
|
except AuthNoTokenException:
|
|
349
341
|
raise
|
|
350
342
|
except Exception as e:
|
tinybird/tb/modules/watch.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import os
|
|
3
2
|
import time
|
|
4
3
|
from pathlib import Path
|
|
@@ -223,13 +222,10 @@ def watch_files(
|
|
|
223
222
|
build_ok: bool,
|
|
224
223
|
) -> None:
|
|
225
224
|
# Handle both sync and async process functions
|
|
226
|
-
|
|
225
|
+
def process_wrapper(files: List[str]) -> None:
|
|
227
226
|
click.echo(FeedbackManager.highlight(message="» Rebuilding project..."))
|
|
228
227
|
time_start = time.time()
|
|
229
|
-
|
|
230
|
-
await process(files, watch=True)
|
|
231
|
-
else:
|
|
232
|
-
process(files, watch=True)
|
|
228
|
+
process(files, watch=True)
|
|
233
229
|
time_end = time.time()
|
|
234
230
|
elapsed_time = time_end - time_start
|
|
235
231
|
click.echo(
|
|
@@ -238,7 +234,7 @@ def watch_files(
|
|
|
238
234
|
)
|
|
239
235
|
shell.reprint_prompt()
|
|
240
236
|
|
|
241
|
-
event_handler = FileChangeHandler(filenames, lambda f:
|
|
237
|
+
event_handler = FileChangeHandler(filenames, lambda f: process_wrapper(f), build_ok)
|
|
242
238
|
observer = Observer()
|
|
243
239
|
|
|
244
240
|
observer.schedule(event_handler, path=str(project.path), recursive=True)
|