tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev236__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +22 -18
- tinybird/tb/modules/local_common.py +13 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev236.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/top_level.txt +0 -0
tinybird/tb/config.py
CHANGED
|
@@ -3,7 +3,6 @@ from os import environ, getcwd
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
from typing import Any, Dict, Optional
|
|
5
5
|
|
|
6
|
-
import aiofiles
|
|
7
6
|
import click
|
|
8
7
|
|
|
9
8
|
from tinybird.tb import __cli__
|
|
@@ -43,7 +42,7 @@ CLOUD_HOSTS = {
|
|
|
43
42
|
|
|
44
43
|
|
|
45
44
|
def get_config(
|
|
46
|
-
host: str, token: Optional[str], semver: Optional[str] = None, config_file: Optional[str] = None
|
|
45
|
+
host: Optional[str], token: Optional[str], semver: Optional[str] = None, config_file: Optional[str] = None
|
|
47
46
|
) -> Dict[str, Any]:
|
|
48
47
|
if host:
|
|
49
48
|
host = host.rstrip("/")
|
|
@@ -72,10 +71,10 @@ def get_config(
|
|
|
72
71
|
return config
|
|
73
72
|
|
|
74
73
|
|
|
75
|
-
|
|
74
|
+
def write_config(config: Dict[str, Any], dest_file: str = ".tinyb") -> None:
|
|
76
75
|
config_file = Path(getcwd()) / dest_file
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
with open(config_file, "w") as file:
|
|
77
|
+
file.write(json.dumps(config, indent=4, sort_keys=True))
|
|
79
78
|
|
|
80
79
|
|
|
81
80
|
def get_display_cloud_host(api_host: str) -> str:
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import json
|
|
3
2
|
import logging
|
|
4
3
|
import threading
|
|
@@ -222,19 +221,17 @@ def append_fixture(
|
|
|
222
221
|
url: str,
|
|
223
222
|
):
|
|
224
223
|
# Append fixtures only if the datasource is empty
|
|
225
|
-
data =
|
|
224
|
+
data = tb_client._req(f"/v0/datasources/{datasource_name}")
|
|
226
225
|
if data.get("statistics", {}).get("row_count", 0) > 0:
|
|
227
226
|
return
|
|
228
227
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
silent=True,
|
|
237
|
-
)
|
|
228
|
+
push_data(
|
|
229
|
+
tb_client,
|
|
230
|
+
datasource_name,
|
|
231
|
+
url,
|
|
232
|
+
mode="append",
|
|
233
|
+
concurrency=1,
|
|
234
|
+
silent=True,
|
|
238
235
|
)
|
|
239
236
|
|
|
240
237
|
|
|
@@ -253,7 +250,7 @@ def rebuild_fixture(project: Project, tb_client: TinyB, fixture: str) -> None:
|
|
|
253
250
|
ds_path = datasources_path / f"{ds_name}.datasource"
|
|
254
251
|
|
|
255
252
|
if ds_path.exists():
|
|
256
|
-
|
|
253
|
+
tb_client.datasource_truncate(ds_name)
|
|
257
254
|
append_fixture(tb_client, ds_name, str(fixture_path))
|
|
258
255
|
except Exception as e:
|
|
259
256
|
click.echo(FeedbackManager.error_exception(error=e))
|
|
@@ -271,7 +268,7 @@ def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
|
|
|
271
268
|
|
|
272
269
|
sql = f"SELECT * FROM {table_name} FORMAT JSON"
|
|
273
270
|
|
|
274
|
-
res =
|
|
271
|
+
res = tb_client.query(sql, pipeline=pipeline)
|
|
275
272
|
print_table_formatted(res, table_name)
|
|
276
273
|
if Project.get_pipe_type(filename) == "endpoint":
|
|
277
274
|
example_params = {
|
|
@@ -280,7 +277,7 @@ def show_data(tb_client: TinyB, filename: str, diff: Optional[str] = None):
|
|
|
280
277
|
"q": "",
|
|
281
278
|
"token": tb_client.token,
|
|
282
279
|
}
|
|
283
|
-
endpoint_url =
|
|
280
|
+
endpoint_url = tb_client._req(f"/examples/query.http?{urlencode(example_params)}")
|
|
284
281
|
if endpoint_url:
|
|
285
282
|
endpoint_url = endpoint_url.replace("http://localhost:8001", tb_client.host)
|
|
286
283
|
click.echo(FeedbackManager.gray(message="\nTest endpoint at ") + FeedbackManager.info(message=endpoint_url))
|
|
@@ -387,7 +384,7 @@ def rebuild_fixture_sql(project: Project, tb_client: TinyB, sql_file: str) -> Pa
|
|
|
387
384
|
fixture_format = current_fixture_path.suffix.lstrip(".") if current_fixture_path else "ndjson"
|
|
388
385
|
sql = sql_path.read_text()
|
|
389
386
|
sql_format = "CSV" if fixture_format == "csv" else "JSON"
|
|
390
|
-
result =
|
|
387
|
+
result = tb_client.query(f"{sql} FORMAT {sql_format}")
|
|
391
388
|
data = result.get("data", [])
|
|
392
389
|
return persist_fixture(datasource_name, data, project.folder, format=fixture_format)
|
|
393
390
|
|
|
@@ -432,26 +429,26 @@ def dev_cloud(
|
|
|
432
429
|
tb_client: TinyB = config.get_client()
|
|
433
430
|
context.disable_template_security_validation.set(True)
|
|
434
431
|
|
|
435
|
-
|
|
432
|
+
def process(filenames: List[str], watch: bool = False):
|
|
436
433
|
datafiles = [f for f in filenames if f.endswith(".datasource") or f.endswith(".pipe")]
|
|
437
434
|
if len(datafiles) > 0:
|
|
438
435
|
check_filenames(filenames=datafiles)
|
|
439
|
-
|
|
436
|
+
folder_playground(
|
|
440
437
|
project, config, tb_client, filenames=datafiles, is_internal=False, current_ws=None, local_ws=None
|
|
441
438
|
)
|
|
442
439
|
if len(filenames) > 0 and watch:
|
|
443
440
|
filename = filenames[0]
|
|
444
|
-
|
|
441
|
+
build_and_print_resource(config, tb_client, filename)
|
|
445
442
|
|
|
446
443
|
datafiles = project.get_project_files()
|
|
447
444
|
filenames = datafiles
|
|
448
445
|
|
|
449
|
-
|
|
446
|
+
def build_once(filenames: List[str]):
|
|
450
447
|
ok = False
|
|
451
448
|
try:
|
|
452
449
|
click.echo(FeedbackManager.highlight(message="» Building project...\n"))
|
|
453
450
|
time_start = time.time()
|
|
454
|
-
|
|
451
|
+
process(filenames=filenames, watch=False)
|
|
455
452
|
time_end = time.time()
|
|
456
453
|
elapsed_time = time_end - time_start
|
|
457
454
|
|
|
@@ -469,7 +466,7 @@ def dev_cloud(
|
|
|
469
466
|
ok = False
|
|
470
467
|
return ok
|
|
471
468
|
|
|
472
|
-
build_ok =
|
|
469
|
+
build_ok = build_once(filenames)
|
|
473
470
|
|
|
474
471
|
shell = Shell(project=project, tb_client=tb_client, playground=True)
|
|
475
472
|
click.echo(FeedbackManager.gray(message="\nWatching for changes..."))
|
|
@@ -480,7 +477,7 @@ def dev_cloud(
|
|
|
480
477
|
shell.run()
|
|
481
478
|
|
|
482
479
|
|
|
483
|
-
|
|
480
|
+
def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename: str):
|
|
484
481
|
resource_path = Path(filename)
|
|
485
482
|
name = resource_path.stem
|
|
486
483
|
playground_name = name if filename.endswith(".pipe") else None
|
|
@@ -488,7 +485,7 @@ async def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename
|
|
|
488
485
|
user_client.token = config.get_user_token() or ""
|
|
489
486
|
cli_params = {}
|
|
490
487
|
cli_params["workspace_id"] = config.get("id", None)
|
|
491
|
-
data =
|
|
488
|
+
data = user_client._req(f"/v0/playgrounds?{urlencode(cli_params)}")
|
|
492
489
|
playgrounds = data["playgrounds"]
|
|
493
490
|
playground = next((p for p in playgrounds if p["name"] == (f"{playground_name}" + "__tb__playground")), None)
|
|
494
491
|
if not playground:
|
|
@@ -498,7 +495,7 @@ async def build_and_print_resource(config: CLIConfig, tb_client: TinyB, filename
|
|
|
498
495
|
if not last_node:
|
|
499
496
|
return
|
|
500
497
|
node_sql = last_node["sql"]
|
|
501
|
-
res =
|
|
498
|
+
res = tb_client.query(f"{node_sql} FORMAT JSON", playground=playground_id)
|
|
502
499
|
print_table_formatted(res, name)
|
|
503
500
|
|
|
504
501
|
|
tinybird/tb/modules/cicd.py
CHANGED
|
@@ -233,7 +233,7 @@ class GitLabCICDGenerator(CICDGeneratorBase):
|
|
|
233
233
|
]
|
|
234
234
|
|
|
235
235
|
|
|
236
|
-
|
|
236
|
+
def init_cicd(
|
|
237
237
|
path: Optional[str] = None,
|
|
238
238
|
data_project_dir: Optional[str] = None,
|
|
239
239
|
):
|
|
@@ -250,7 +250,7 @@ async def init_cicd(
|
|
|
250
250
|
click.echo(warning_message)
|
|
251
251
|
|
|
252
252
|
|
|
253
|
-
|
|
253
|
+
def check_cicd_exists(path: Optional[str] = None) -> Optional[Provider]:
|
|
254
254
|
path = path if path else getcwd()
|
|
255
255
|
for provider in Provider:
|
|
256
256
|
generator = CICDGeneratorBase.build_generator(provider.name)
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -12,7 +12,6 @@ from os import environ, getcwd
|
|
|
12
12
|
from pathlib import Path
|
|
13
13
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
14
14
|
|
|
15
|
-
import aiofiles
|
|
16
15
|
import click
|
|
17
16
|
import humanfriendly
|
|
18
17
|
from click import Context
|
|
@@ -27,7 +26,6 @@ from tinybird.tb.modules.common import (
|
|
|
27
26
|
CatchAuthExceptions,
|
|
28
27
|
CLIException,
|
|
29
28
|
_get_tb_client,
|
|
30
|
-
coro,
|
|
31
29
|
echo_json,
|
|
32
30
|
echo_safe_format_table,
|
|
33
31
|
force_echo,
|
|
@@ -79,8 +77,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
|
|
|
79
77
|
@click.option("--max-depth", type=int, default=3, help="Maximum depth of the project files.")
|
|
80
78
|
@click.version_option(version=VERSION)
|
|
81
79
|
@click.pass_context
|
|
82
|
-
|
|
83
|
-
async def cli(
|
|
80
|
+
def cli(
|
|
84
81
|
ctx: Context,
|
|
85
82
|
debug: bool,
|
|
86
83
|
token: str,
|
|
@@ -109,7 +106,7 @@ async def cli(
|
|
|
109
106
|
click.echo(FeedbackManager.warning_disabled_ssl_checks())
|
|
110
107
|
|
|
111
108
|
if not environ.get("PYTEST", None) and version_warning and not token:
|
|
112
|
-
latest_version =
|
|
109
|
+
latest_version = CheckPypi().get_latest_version()
|
|
113
110
|
if latest_version:
|
|
114
111
|
if "x.y.z" in CURRENT_VERSION:
|
|
115
112
|
click.echo(FeedbackManager.warning_development_cli())
|
|
@@ -136,7 +133,7 @@ async def cli(
|
|
|
136
133
|
if user_token:
|
|
137
134
|
config_temp.set_user_token(user_token)
|
|
138
135
|
if token or host or user_token:
|
|
139
|
-
|
|
136
|
+
try_update_config_with_remote(config_temp, auto_persist=False, raise_on_errors=False)
|
|
140
137
|
|
|
141
138
|
# Overwrite token and host with env vars manually, without resorting to click.
|
|
142
139
|
#
|
|
@@ -150,7 +147,7 @@ async def cli(
|
|
|
150
147
|
if not user_token and "TB_USER_TOKEN" in os.environ:
|
|
151
148
|
user_token = os.environ.get("TB_USER_TOKEN", "")
|
|
152
149
|
|
|
153
|
-
config =
|
|
150
|
+
config = get_config(host, token, user_token=user_token, config_file=config_temp._path)
|
|
154
151
|
client = _get_tb_client(config.get("token", None), config["host"])
|
|
155
152
|
|
|
156
153
|
# Calculate project folder path properly
|
|
@@ -169,10 +166,9 @@ async def cli(
|
|
|
169
166
|
# If they have passed a token or host as parameter and it's different that record in .tinyb, refresh the workspace id
|
|
170
167
|
if token or host:
|
|
171
168
|
try:
|
|
172
|
-
workspace =
|
|
169
|
+
workspace = client.workspace_info(version="v1")
|
|
173
170
|
config["id"] = workspace.get("id", "")
|
|
174
171
|
config["name"] = workspace.get("name", "")
|
|
175
|
-
# If we can not get this info, we continue with the id on the file
|
|
176
172
|
except (AuthNoTokenException, AuthException):
|
|
177
173
|
pass
|
|
178
174
|
|
|
@@ -183,7 +179,7 @@ async def cli(
|
|
|
183
179
|
if "--help" in sys.argv or "-h" in sys.argv:
|
|
184
180
|
return
|
|
185
181
|
|
|
186
|
-
client =
|
|
182
|
+
client = create_ctx_client(ctx, config, cloud, staging)
|
|
187
183
|
|
|
188
184
|
if client:
|
|
189
185
|
ctx.ensure_object(dict)["client"] = client
|
|
@@ -197,14 +193,13 @@ async def cli(
|
|
|
197
193
|
@click.option("-f", "--force", is_flag=True, default=False, help="Override existing files")
|
|
198
194
|
@click.option("--fmt", is_flag=True, default=False, help="Format files before saving")
|
|
199
195
|
@click.pass_context
|
|
200
|
-
|
|
201
|
-
async def pull(ctx: Context, force: bool, fmt: bool) -> None:
|
|
196
|
+
def pull(ctx: Context, force: bool, fmt: bool) -> None:
|
|
202
197
|
"""Retrieve latest versions for project files from Tinybird."""
|
|
203
198
|
|
|
204
199
|
client = ctx.ensure_object(dict)["client"]
|
|
205
200
|
project = ctx.ensure_object(dict)["project"]
|
|
206
201
|
|
|
207
|
-
return
|
|
202
|
+
return folder_pull(client, project.path, force, fmt=fmt)
|
|
208
203
|
|
|
209
204
|
|
|
210
205
|
@cli.command()
|
|
@@ -215,8 +210,7 @@ async def pull(ctx: Context, force: bool, fmt: bool) -> None:
|
|
|
215
210
|
@click.option("--node", default=None, help="The NODE name")
|
|
216
211
|
@click.option("--stats/--no-stats", default=False, help="Show query stats")
|
|
217
212
|
@click.pass_context
|
|
218
|
-
|
|
219
|
-
async def sql(
|
|
213
|
+
def sql(
|
|
220
214
|
ctx: Context,
|
|
221
215
|
query: str,
|
|
222
216
|
rows_limit: int,
|
|
@@ -244,15 +238,13 @@ async def sql(
|
|
|
244
238
|
raise CLIException(FeedbackManager.error_invalid_query())
|
|
245
239
|
if q.startswith("delete"):
|
|
246
240
|
raise CLIException(FeedbackManager.error_invalid_query())
|
|
247
|
-
res =
|
|
248
|
-
f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT {req_format}", pipeline=pipeline
|
|
249
|
-
)
|
|
241
|
+
res = client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT {req_format}", pipeline=pipeline)
|
|
250
242
|
elif pipe and node:
|
|
251
243
|
filenames = [pipe]
|
|
252
244
|
|
|
253
245
|
# build graph to get new versions for all the files involved in the query
|
|
254
246
|
# dependencies need to be processed always to get the versions
|
|
255
|
-
dependencies_graph =
|
|
247
|
+
dependencies_graph = build_graph(
|
|
256
248
|
filenames,
|
|
257
249
|
client,
|
|
258
250
|
dir_path=".",
|
|
@@ -266,9 +258,7 @@ async def sql(
|
|
|
266
258
|
if _node["params"]["name"].lower() == node.lower():
|
|
267
259
|
query = "".join(_node["sql"])
|
|
268
260
|
pipeline = pipe.split("/")[-1].split(".pipe")[0]
|
|
269
|
-
res =
|
|
270
|
-
f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT {req_format}", pipeline=pipeline
|
|
271
|
-
)
|
|
261
|
+
res = client.query(f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT {req_format}", pipeline=pipeline)
|
|
272
262
|
|
|
273
263
|
except AuthNoTokenException:
|
|
274
264
|
raise
|
|
@@ -280,7 +270,7 @@ async def sql(
|
|
|
280
270
|
|
|
281
271
|
if stats:
|
|
282
272
|
stats_query = f"SELECT * FROM ({query}) LIMIT {rows_limit} FORMAT JSON"
|
|
283
|
-
stats_res =
|
|
273
|
+
stats_res = client.query(stats_query, pipeline=pipeline)
|
|
284
274
|
stats_dict = stats_res["statistics"]
|
|
285
275
|
seconds = stats_dict["elapsed"]
|
|
286
276
|
rows_read = humanfriendly.format_number(stats_dict["rows_read"])
|
|
@@ -356,7 +346,7 @@ def __hide_click_output() -> None:
|
|
|
356
346
|
click.secho = silent_secho # type: ignore
|
|
357
347
|
|
|
358
348
|
|
|
359
|
-
|
|
349
|
+
def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, staging: bool):
|
|
360
350
|
commands_without_ctx_client = [
|
|
361
351
|
"auth",
|
|
362
352
|
"check",
|
|
@@ -404,7 +394,7 @@ async def create_ctx_client(ctx: Context, config: Dict[str, Any], cloud: bool, s
|
|
|
404
394
|
test = command in command_always_test
|
|
405
395
|
if not local and command not in commands_always_local and command:
|
|
406
396
|
click.echo(FeedbackManager.gray(message="Running against Tinybird Local"))
|
|
407
|
-
return
|
|
397
|
+
return get_tinybird_local_client(config, test=test, staging=staging)
|
|
408
398
|
|
|
409
399
|
|
|
410
400
|
def get_target_env(cloud: bool) -> str:
|
|
@@ -413,7 +403,7 @@ def get_target_env(cloud: bool) -> str:
|
|
|
413
403
|
return "local"
|
|
414
404
|
|
|
415
405
|
|
|
416
|
-
|
|
406
|
+
def get_config(
|
|
417
407
|
host: str,
|
|
418
408
|
token: Optional[str],
|
|
419
409
|
user_token: Optional[str],
|
|
@@ -425,8 +415,8 @@ async def get_config(
|
|
|
425
415
|
|
|
426
416
|
config = {}
|
|
427
417
|
try:
|
|
428
|
-
|
|
429
|
-
res =
|
|
418
|
+
with open(config_file or Path(getcwd()) / ".tinyb") as file:
|
|
419
|
+
res = file.read()
|
|
430
420
|
config = json.loads(res)
|
|
431
421
|
except OSError:
|
|
432
422
|
pass
|