tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev235__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +6 -12
- tinybird/tb/modules/local_common.py +4 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev235.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/top_level.txt +0 -0
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import asyncio
|
|
7
6
|
import json
|
|
8
7
|
import os
|
|
9
8
|
import re
|
|
9
|
+
import time
|
|
10
10
|
import uuid
|
|
11
11
|
from datetime import datetime
|
|
12
12
|
from pathlib import Path
|
|
@@ -20,14 +20,12 @@ from click import Context
|
|
|
20
20
|
|
|
21
21
|
from tinybird.datafile.common import get_name_version
|
|
22
22
|
from tinybird.prompts import quarantine_prompt
|
|
23
|
-
from tinybird.syncasync import sync_to_async
|
|
24
23
|
from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
|
|
25
24
|
from tinybird.tb.modules.build import process as build_project
|
|
26
25
|
from tinybird.tb.modules.cli import cli
|
|
27
26
|
from tinybird.tb.modules.common import (
|
|
28
27
|
_analyze,
|
|
29
28
|
analyze_file,
|
|
30
|
-
coro,
|
|
31
29
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
32
30
|
get_format_from_filename_or_url,
|
|
33
31
|
load_connector_config,
|
|
@@ -66,12 +64,11 @@ def datasource(ctx):
|
|
|
66
64
|
help="Force a type of the output",
|
|
67
65
|
)
|
|
68
66
|
@click.pass_context
|
|
69
|
-
|
|
70
|
-
async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
67
|
+
def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
71
68
|
"""List data sources"""
|
|
72
69
|
|
|
73
70
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
74
|
-
ds =
|
|
71
|
+
ds = client.datasources()
|
|
75
72
|
columns = ["shared from", "name", "row_count", "size", "created at", "updated at", "connection"]
|
|
76
73
|
table_human_readable = []
|
|
77
74
|
table_machine_readable = []
|
|
@@ -134,8 +131,7 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
134
131
|
@click.option("--events", type=str, help="Events to append data from")
|
|
135
132
|
@click.option("--concurrency", help="How many files to submit concurrently", default=1, hidden=True)
|
|
136
133
|
@click.pass_context
|
|
137
|
-
|
|
138
|
-
async def datasource_append(
|
|
134
|
+
def datasource_append(
|
|
139
135
|
ctx: Context,
|
|
140
136
|
datasource_name: str,
|
|
141
137
|
data: Optional[str],
|
|
@@ -194,7 +190,7 @@ async def datasource_append(
|
|
|
194
190
|
else:
|
|
195
191
|
tip = "Did you deploy your project? Run `tb --cloud deploy` first."
|
|
196
192
|
|
|
197
|
-
datasources =
|
|
193
|
+
datasources = client.datasources()
|
|
198
194
|
if not datasources:
|
|
199
195
|
raise CLIDatasourceException(FeedbackManager.error(message=f"No data sources found. {tip}"))
|
|
200
196
|
|
|
@@ -265,7 +261,7 @@ async def datasource_append(
|
|
|
265
261
|
|
|
266
262
|
if events:
|
|
267
263
|
click.echo(FeedbackManager.highlight(message=f"\n» Sending events to {datasource_name}"))
|
|
268
|
-
response =
|
|
264
|
+
response = requests.post(
|
|
269
265
|
f"{client.host}/v0/events?name={datasource_name}",
|
|
270
266
|
headers={"Authorization": f"Bearer {client.token}"},
|
|
271
267
|
data=events,
|
|
@@ -290,12 +286,12 @@ async def datasource_append(
|
|
|
290
286
|
message=f"✗ {quarantined_rows} row{'' if quarantined_rows == 1 else 's'} went to quarantine"
|
|
291
287
|
)
|
|
292
288
|
)
|
|
293
|
-
|
|
289
|
+
analyze_quarantine(datasource_name, project, client)
|
|
294
290
|
return
|
|
295
291
|
else:
|
|
296
292
|
click.echo(FeedbackManager.highlight(message=f"\n» Appending data to {datasource_name}"))
|
|
297
293
|
try:
|
|
298
|
-
|
|
294
|
+
push_data(
|
|
299
295
|
client,
|
|
300
296
|
datasource_name,
|
|
301
297
|
data,
|
|
@@ -307,7 +303,7 @@ async def datasource_append(
|
|
|
307
303
|
is_quarantined = "quarantine" in str(e)
|
|
308
304
|
click.echo(FeedbackManager.error(message="✗ " + str(e)))
|
|
309
305
|
if is_quarantined:
|
|
310
|
-
|
|
306
|
+
analyze_quarantine(datasource_name, project, client)
|
|
311
307
|
return
|
|
312
308
|
else:
|
|
313
309
|
raise e
|
|
@@ -320,8 +316,7 @@ async def datasource_append(
|
|
|
320
316
|
@click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
|
|
321
317
|
@click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
|
|
322
318
|
@click.pass_context
|
|
323
|
-
|
|
324
|
-
async def datasource_replace(
|
|
319
|
+
def datasource_replace(
|
|
325
320
|
ctx: Context,
|
|
326
321
|
datasource_name,
|
|
327
322
|
url,
|
|
@@ -340,7 +335,7 @@ async def datasource_replace(
|
|
|
340
335
|
if skip_incompatible_partition_key:
|
|
341
336
|
replace_options.add("skip_incompatible_partition_key")
|
|
342
337
|
client: TinyB = ctx.obj["client"]
|
|
343
|
-
|
|
338
|
+
push_data(
|
|
344
339
|
client,
|
|
345
340
|
datasource_name,
|
|
346
341
|
url,
|
|
@@ -359,8 +354,7 @@ async def datasource_replace(
|
|
|
359
354
|
hidden=True,
|
|
360
355
|
)
|
|
361
356
|
@click.pass_context
|
|
362
|
-
|
|
363
|
-
async def datasource_analyze(ctx, url_or_file, connector):
|
|
357
|
+
def datasource_analyze(ctx, url_or_file, connector):
|
|
364
358
|
"""Analyze a URL or a file before creating a new data source"""
|
|
365
359
|
client = ctx.obj["client"]
|
|
366
360
|
|
|
@@ -379,7 +373,7 @@ async def datasource_analyze(ctx, url_or_file, connector):
|
|
|
379
373
|
for t in data:
|
|
380
374
|
click.echo(FeedbackManager.info_datasource_row(row=row_format.format(*[str(element) for element in t])))
|
|
381
375
|
|
|
382
|
-
analysis, _ =
|
|
376
|
+
analysis, _ = _analyze(
|
|
383
377
|
url_or_file, client, format=get_format_from_filename_or_url(url_or_file), connector=_connector
|
|
384
378
|
)
|
|
385
379
|
|
|
@@ -418,14 +412,13 @@ async def datasource_analyze(ctx, url_or_file, connector):
|
|
|
418
412
|
"--cascade", is_flag=True, default=False, help="Truncate dependent DS attached in cascade to the given DS"
|
|
419
413
|
)
|
|
420
414
|
@click.pass_context
|
|
421
|
-
|
|
422
|
-
async def datasource_truncate(ctx, datasource_name, yes, cascade):
|
|
415
|
+
def datasource_truncate(ctx, datasource_name, yes, cascade):
|
|
423
416
|
"""Truncate a data source"""
|
|
424
417
|
|
|
425
418
|
client = ctx.obj["client"]
|
|
426
419
|
if yes or click.confirm(FeedbackManager.warning_confirm_truncate_datasource(datasource=datasource_name)):
|
|
427
420
|
try:
|
|
428
|
-
|
|
421
|
+
client.datasource_truncate(datasource_name)
|
|
429
422
|
except AuthNoTokenException:
|
|
430
423
|
raise
|
|
431
424
|
except DoesNotExistException:
|
|
@@ -437,7 +430,7 @@ async def datasource_truncate(ctx, datasource_name, yes, cascade):
|
|
|
437
430
|
|
|
438
431
|
if cascade:
|
|
439
432
|
try:
|
|
440
|
-
ds_cascade_dependencies =
|
|
433
|
+
ds_cascade_dependencies = client.datasource_dependencies(
|
|
441
434
|
no_deps=False,
|
|
442
435
|
match=None,
|
|
443
436
|
pipe=None,
|
|
@@ -454,7 +447,7 @@ async def datasource_truncate(ctx, datasource_name, yes, cascade):
|
|
|
454
447
|
for cascade_ds in cascade_dependent_ds:
|
|
455
448
|
if yes or click.confirm(FeedbackManager.warning_confirm_truncate_datasource(datasource=cascade_ds)):
|
|
456
449
|
try:
|
|
457
|
-
|
|
450
|
+
client.datasource_truncate(cascade_ds)
|
|
458
451
|
except DoesNotExistException:
|
|
459
452
|
raise CLIDatasourceException(
|
|
460
453
|
FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name)
|
|
@@ -471,8 +464,7 @@ async def datasource_truncate(ctx, datasource_name, yes, cascade):
|
|
|
471
464
|
@click.option("--wait", is_flag=True, default=False, help="Wait for delete job to finish, disabled by default")
|
|
472
465
|
@click.option("--dry-run", is_flag=True, default=False, help="Run the command without deleting anything")
|
|
473
466
|
@click.pass_context
|
|
474
|
-
|
|
475
|
-
async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait, dry_run):
|
|
467
|
+
def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait, dry_run):
|
|
476
468
|
"""
|
|
477
469
|
Delete rows from a datasource
|
|
478
470
|
|
|
@@ -492,7 +484,7 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
|
|
|
492
484
|
)
|
|
493
485
|
):
|
|
494
486
|
try:
|
|
495
|
-
res =
|
|
487
|
+
res = client.datasource_delete_rows(datasource_name, sql_condition, dry_run)
|
|
496
488
|
if dry_run:
|
|
497
489
|
click.echo(
|
|
498
490
|
FeedbackManager.success_dry_run_delete_rows_datasource(
|
|
@@ -515,7 +507,7 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
|
|
|
515
507
|
i = 0
|
|
516
508
|
while True:
|
|
517
509
|
try:
|
|
518
|
-
res =
|
|
510
|
+
res = client._req(f"v0/jobs/{job_id}")
|
|
519
511
|
except Exception:
|
|
520
512
|
raise CLIDatasourceException(FeedbackManager.error_job_status(url=job_url))
|
|
521
513
|
if res["status"] == "done":
|
|
@@ -529,7 +521,7 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
|
|
|
529
521
|
elif res["status"] == "error":
|
|
530
522
|
print("\n") # noqa: T201
|
|
531
523
|
raise CLIDatasourceException(FeedbackManager.error_exception(error=res["error"]))
|
|
532
|
-
|
|
524
|
+
time.sleep(1)
|
|
533
525
|
i += 1
|
|
534
526
|
progress_line(i)
|
|
535
527
|
|
|
@@ -551,8 +543,7 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
|
|
|
551
543
|
@click.argument("datasource")
|
|
552
544
|
@click.option("--limit", type=int, default=5, help="Limit the number of rows to return")
|
|
553
545
|
@click.pass_context
|
|
554
|
-
|
|
555
|
-
async def datasource_data(ctx: Context, datasource: str, limit: int):
|
|
546
|
+
def datasource_data(ctx: Context, datasource: str, limit: int):
|
|
556
547
|
"""Print data returned by an endpoint
|
|
557
548
|
|
|
558
549
|
Syntax: tb datasource data <datasource_name>
|
|
@@ -560,7 +551,7 @@ async def datasource_data(ctx: Context, datasource: str, limit: int):
|
|
|
560
551
|
|
|
561
552
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
562
553
|
try:
|
|
563
|
-
res =
|
|
554
|
+
res = client.query(f"SELECT * FROM {datasource} LIMIT {limit} FORMAT JSON")
|
|
564
555
|
except AuthNoTokenException:
|
|
565
556
|
raise
|
|
566
557
|
except Exception as e:
|
|
@@ -587,8 +578,7 @@ async def datasource_data(ctx: Context, datasource: str, limit: int):
|
|
|
587
578
|
@click.option("--where", type=str, default=None, help="Condition to filter data")
|
|
588
579
|
@click.option("--target", type=str, help="Target file path (default: datasource_name.{format})")
|
|
589
580
|
@click.pass_context
|
|
590
|
-
|
|
591
|
-
async def datasource_export(
|
|
581
|
+
def datasource_export(
|
|
592
582
|
ctx: Context,
|
|
593
583
|
datasource: str,
|
|
594
584
|
format_: str,
|
|
@@ -617,7 +607,7 @@ async def datasource_export(
|
|
|
617
607
|
else:
|
|
618
608
|
query += " FORMAT JSONEachRow"
|
|
619
609
|
|
|
620
|
-
res =
|
|
610
|
+
res = client.query(query)
|
|
621
611
|
|
|
622
612
|
target_path = persist_fixture(datasource, res, project.folder, format=format_, target=target)
|
|
623
613
|
file_size = os.path.getsize(target_path)
|
|
@@ -636,16 +626,15 @@ async def datasource_export(
|
|
|
636
626
|
@click.argument("datasource_name")
|
|
637
627
|
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
638
628
|
@click.pass_context
|
|
639
|
-
|
|
640
|
-
async def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
|
|
629
|
+
def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
|
|
641
630
|
try:
|
|
642
631
|
client: TinyB = ctx.obj["client"]
|
|
643
|
-
ds =
|
|
632
|
+
ds = client.get_datasource(datasource_name)
|
|
644
633
|
|
|
645
634
|
warning_message = FeedbackManager.warning_datasource_sync_bucket(datasource=datasource_name)
|
|
646
635
|
|
|
647
636
|
if yes or click.confirm(warning_message):
|
|
648
|
-
|
|
637
|
+
client.datasource_sync(ds["id"])
|
|
649
638
|
click.echo(FeedbackManager.success_sync_datasource(datasource=datasource_name))
|
|
650
639
|
except AuthNoTokenException:
|
|
651
640
|
raise
|
|
@@ -664,8 +653,7 @@ async def datasource_sync(ctx: Context, datasource_name: str, yes: bool):
|
|
|
664
653
|
@click.option("--gcs", is_flag=True, default=False, help="Create a data source from a GCS connection")
|
|
665
654
|
@click.option("--kafka", is_flag=True, default=False, help="Create a data source from a Kafka connection")
|
|
666
655
|
@click.pass_context
|
|
667
|
-
|
|
668
|
-
async def datasource_create(
|
|
656
|
+
def datasource_create(
|
|
669
657
|
ctx: Context,
|
|
670
658
|
name: str,
|
|
671
659
|
blank: bool,
|
|
@@ -781,7 +769,7 @@ async def datasource_create(
|
|
|
781
769
|
raise Exception("This action requires authentication. Run 'tb login' first.")
|
|
782
770
|
project_config = CLIConfig.get_project_config()
|
|
783
771
|
tb_client: TinyB = project_config.get_client(token=config.get("token"), host=config.get("host"))
|
|
784
|
-
|
|
772
|
+
create_resources_from_prompt(tb_client, user_token, prompt, project, feature="tb_datasource_create")
|
|
785
773
|
click.echo(FeedbackManager.success(message="✓ .datasource created!"))
|
|
786
774
|
return
|
|
787
775
|
|
|
@@ -832,9 +820,9 @@ async def datasource_create(
|
|
|
832
820
|
sasl_mechanism,
|
|
833
821
|
security_protocol,
|
|
834
822
|
topics,
|
|
835
|
-
) =
|
|
823
|
+
) = connection_create_kafka(ctx)
|
|
836
824
|
elif datasource_type == "s3":
|
|
837
|
-
|
|
825
|
+
generate_aws_iamrole_connection_file_with_secret(
|
|
838
826
|
connection_name,
|
|
839
827
|
service="s3",
|
|
840
828
|
role_arn_secret_name="S3_ARN",
|
|
@@ -843,7 +831,7 @@ async def datasource_create(
|
|
|
843
831
|
with_default_secret=True,
|
|
844
832
|
)
|
|
845
833
|
elif datasource_type == "gcs":
|
|
846
|
-
|
|
834
|
+
generate_gcs_connection_file_with_secrets(
|
|
847
835
|
connection_name,
|
|
848
836
|
service="gcs",
|
|
849
837
|
svc_account_creds="GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
|
|
@@ -896,7 +884,7 @@ async def datasource_create(
|
|
|
896
884
|
path = Path(file)
|
|
897
885
|
|
|
898
886
|
data_format = path.suffix.lstrip(".")
|
|
899
|
-
ds_content =
|
|
887
|
+
ds_content = analyze_file(str(path), client, format=data_format)
|
|
900
888
|
default_name = normalize_datasource_name(path.stem)
|
|
901
889
|
name = name or click.prompt(
|
|
902
890
|
FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
|
|
@@ -908,7 +896,7 @@ async def datasource_create(
|
|
|
908
896
|
if not url:
|
|
909
897
|
url = click.prompt(FeedbackManager.highlight(message="? URL"))
|
|
910
898
|
format = url.split(".")[-1]
|
|
911
|
-
ds_content =
|
|
899
|
+
ds_content = analyze_file(url, client, format)
|
|
912
900
|
default_name = normalize_datasource_name(Path(url).stem)
|
|
913
901
|
name = name or click.prompt(
|
|
914
902
|
FeedbackManager.highlight(message=f"? Data source name [{default_name}]"),
|
|
@@ -925,7 +913,7 @@ async def datasource_create(
|
|
|
925
913
|
)
|
|
926
914
|
|
|
927
915
|
if datasource_type == "kafka":
|
|
928
|
-
connections =
|
|
916
|
+
connections = client.connections("kafka")
|
|
929
917
|
kafka_connection_files = project.get_kafka_connection_files()
|
|
930
918
|
|
|
931
919
|
# if we have no topics from before and no connections, we need to build the project
|
|
@@ -941,13 +929,13 @@ async def datasource_create(
|
|
|
941
929
|
click.echo(FeedbackManager.gray(message="» Building project..."))
|
|
942
930
|
build_project(project=project, tb_client=client, watch=False, silent=True)
|
|
943
931
|
click.echo(FeedbackManager.success(message="✓ Build completed!"))
|
|
944
|
-
connections =
|
|
932
|
+
connections = client.connections("kafka")
|
|
945
933
|
|
|
946
934
|
connection_id = next((c["id"] for c in connections if c["name"] == connection), connection)
|
|
947
935
|
|
|
948
936
|
if not topics:
|
|
949
937
|
try:
|
|
950
|
-
topics =
|
|
938
|
+
topics = client.kafka_list_topics(connection_id) if connection_id else []
|
|
951
939
|
except Exception:
|
|
952
940
|
topics = []
|
|
953
941
|
|
|
@@ -975,7 +963,7 @@ KAFKA_GROUP_ID {group_id}
|
|
|
975
963
|
|
|
976
964
|
if datasource_type == "s3":
|
|
977
965
|
if not connection:
|
|
978
|
-
connections =
|
|
966
|
+
connections = client.connections("s3")
|
|
979
967
|
connection = next((c["name"] for c in connections if c["name"] == connection), connection)
|
|
980
968
|
ds_content += f"""
|
|
981
969
|
IMPORT_CONNECTION_NAME "{connection}"
|
|
@@ -985,7 +973,7 @@ IMPORT_SCHEDULE "@auto"
|
|
|
985
973
|
|
|
986
974
|
if datasource_type == "gcs":
|
|
987
975
|
if not connection:
|
|
988
|
-
connections =
|
|
976
|
+
connections = client.connections("gcs")
|
|
989
977
|
connection = next((c["name"] for c in connections if c["name"] == connection), connection)
|
|
990
978
|
ds_content += f"""
|
|
991
979
|
IMPORT_CONNECTION_NAME "{connection}"
|
|
@@ -1026,11 +1014,9 @@ def generate_kafka_group_id(topic: str):
|
|
|
1026
1014
|
return f"{topic}_{int(datetime.timestamp(datetime.now()))}"
|
|
1027
1015
|
|
|
1028
1016
|
|
|
1029
|
-
|
|
1017
|
+
def analyze_quarantine(datasource_name: str, project: Project, client: TinyB):
|
|
1030
1018
|
config = CLIConfig.get_project_config()
|
|
1031
|
-
res =
|
|
1032
|
-
f"SELECT * FROM {datasource_name}_quarantine ORDER BY insertion_date DESC LIMIT 1 FORMAT JSON"
|
|
1033
|
-
)
|
|
1019
|
+
res = client.query(f"SELECT * FROM {datasource_name}_quarantine ORDER BY insertion_date DESC LIMIT 1 FORMAT JSON")
|
|
1034
1020
|
quarantine_data = res["data"]
|
|
1035
1021
|
error_message = json.dumps(res["data"])
|
|
1036
1022
|
user_token = config.get_user_token()
|
tinybird/tb/modules/endpoint.py
CHANGED
|
@@ -17,7 +17,7 @@ from click import Context
|
|
|
17
17
|
from tinybird.datafile.common import get_name_version
|
|
18
18
|
from tinybird.tb.client import AuthNoTokenException, DoesNotExistException, TinyB
|
|
19
19
|
from tinybird.tb.modules.cli import cli
|
|
20
|
-
from tinybird.tb.modules.common import
|
|
20
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
|
|
21
21
|
from tinybird.tb.modules.exceptions import CLIPipeException
|
|
22
22
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
23
23
|
|
|
@@ -38,15 +38,14 @@ def endpoint(ctx):
|
|
|
38
38
|
help="Force a type of the output",
|
|
39
39
|
)
|
|
40
40
|
@click.pass_context
|
|
41
|
-
|
|
42
|
-
async def endpoint_ls(ctx: Context, match: str, format_: str):
|
|
41
|
+
def endpoint_ls(ctx: Context, match: str, format_: str):
|
|
43
42
|
"""List endpoints"""
|
|
44
43
|
|
|
45
44
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
46
|
-
pipes =
|
|
45
|
+
pipes = client.pipes(dependencies=False, node_attrs="name", attrs="name,updated_at,endpoint,url")
|
|
47
46
|
endpoints = [p for p in pipes if p.get("endpoint")]
|
|
48
47
|
endpoints = sorted(endpoints, key=lambda p: p["updated_at"])
|
|
49
|
-
tokens =
|
|
48
|
+
tokens = client.tokens()
|
|
50
49
|
columns = ["name", "updated at", "nodes", "url"]
|
|
51
50
|
table_human_readable = []
|
|
52
51
|
table_machine_readable = []
|
|
@@ -80,17 +79,16 @@ async def endpoint_ls(ctx: Context, match: str, format_: str):
|
|
|
80
79
|
@endpoint.command(name="token")
|
|
81
80
|
@click.argument("pipe_name")
|
|
82
81
|
@click.pass_context
|
|
83
|
-
|
|
84
|
-
async def endpoint_token(ctx: click.Context, pipe_name: str):
|
|
82
|
+
def endpoint_token(ctx: click.Context, pipe_name: str):
|
|
85
83
|
"""Retrieve a token to read an endpoint"""
|
|
86
84
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
87
85
|
|
|
88
86
|
try:
|
|
89
|
-
|
|
87
|
+
client.pipe_file(pipe_name)
|
|
90
88
|
except DoesNotExistException:
|
|
91
89
|
raise CLIPipeException(FeedbackManager.error_pipe_does_not_exist(pipe=pipe_name))
|
|
92
90
|
|
|
93
|
-
tokens =
|
|
91
|
+
tokens = client.tokens()
|
|
94
92
|
token = get_endpoint_token(tokens, pipe_name)
|
|
95
93
|
if token:
|
|
96
94
|
click.echo(token)
|
|
@@ -111,8 +109,7 @@ async def endpoint_token(ctx: click.Context, pipe_name: str):
|
|
|
111
109
|
"--format", "format_", type=click.Choice(["json", "csv"], case_sensitive=False), help="Return format (CSV, JSON)"
|
|
112
110
|
)
|
|
113
111
|
@click.pass_context
|
|
114
|
-
|
|
115
|
-
async def endpoint_data(ctx: Context, pipe: str, query: str, format_: str):
|
|
112
|
+
def endpoint_data(ctx: Context, pipe: str, query: str, format_: str):
|
|
116
113
|
"""Print data returned by an endpoint
|
|
117
114
|
|
|
118
115
|
Syntax: tb endpoint data <pipe_name> --param_name value --param2_name value2 ...
|
|
@@ -122,7 +119,7 @@ async def endpoint_data(ctx: Context, pipe: str, query: str, format_: str):
|
|
|
122
119
|
params = {ctx.args[i][2:]: ctx.args[i + 1] for i in range(0, len(ctx.args), 2)}
|
|
123
120
|
req_format = "json" if not format_ else format_.lower()
|
|
124
121
|
try:
|
|
125
|
-
res =
|
|
122
|
+
res = client.pipe_data(pipe, format=req_format, sql=query, params=params)
|
|
126
123
|
except AuthNoTokenException:
|
|
127
124
|
raise
|
|
128
125
|
except Exception as e:
|
|
@@ -158,13 +155,12 @@ async def endpoint_data(ctx: Context, pipe: str, query: str, format_: str):
|
|
|
158
155
|
help="Language used for sending the request. Options: http, python, curl, javascript, rust, go",
|
|
159
156
|
)
|
|
160
157
|
@click.pass_context
|
|
161
|
-
|
|
162
|
-
async def endpoint_url(ctx: Context, pipe: str, language: str):
|
|
158
|
+
def endpoint_url(ctx: Context, pipe: str, language: str):
|
|
163
159
|
"""Print the URL of an endpoint"""
|
|
164
160
|
if language != "http":
|
|
165
161
|
click.echo(FeedbackManager.highlight(message=f"\n» Generating snippet for {language} language"))
|
|
166
162
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
167
|
-
tokens =
|
|
163
|
+
tokens = client.tokens()
|
|
168
164
|
token = get_endpoint_token(tokens, pipe) or client.token
|
|
169
165
|
click.echo(build_endpoint_snippet(client, pipe, token, language, client.staging))
|
|
170
166
|
if language != "http":
|
|
@@ -192,13 +188,12 @@ def get_endpoint_token(tokens: List[Dict[str, Any]], pipe_name: str) -> Optional
|
|
|
192
188
|
help="Force a type of the output. To parse the output, keep in mind to use `tb --no-version-warning endpoint stats` option.",
|
|
193
189
|
)
|
|
194
190
|
@click.pass_context
|
|
195
|
-
|
|
196
|
-
async def endpoint_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: str):
|
|
191
|
+
def endpoint_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: str):
|
|
197
192
|
"""
|
|
198
193
|
Print endpoint stats for the last 7 days
|
|
199
194
|
"""
|
|
200
195
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
201
|
-
all_pipes =
|
|
196
|
+
all_pipes = client.pipes()
|
|
202
197
|
pipes_to_get_stats = []
|
|
203
198
|
pipes_ids: Dict = {}
|
|
204
199
|
|
|
@@ -232,7 +227,7 @@ async def endpoint_stats(ctx: click.Context, pipes: Tuple[str, ...], format_: st
|
|
|
232
227
|
FORMAT JSON
|
|
233
228
|
"""
|
|
234
229
|
|
|
235
|
-
res =
|
|
230
|
+
res = client.query(sql)
|
|
236
231
|
|
|
237
232
|
if res and "error" in res:
|
|
238
233
|
raise CLIPipeException(FeedbackManager.error_exception(error=str(res["error"])))
|
tinybird/tb/modules/info.py
CHANGED
|
@@ -6,7 +6,7 @@ import click
|
|
|
6
6
|
from tinybird.tb.client import TinyB
|
|
7
7
|
from tinybird.tb.config import get_display_cloud_host
|
|
8
8
|
from tinybird.tb.modules.cli import CLIConfig, cli
|
|
9
|
-
from tinybird.tb.modules.common import
|
|
9
|
+
from tinybird.tb.modules.common import echo_json, force_echo, format_robust_table
|
|
10
10
|
from tinybird.tb.modules.exceptions import CLILocalException
|
|
11
11
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
12
12
|
from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_tinybird_local_config
|
|
@@ -15,8 +15,7 @@ from tinybird.tb.modules.project import Project
|
|
|
15
15
|
|
|
16
16
|
@cli.command(name="info")
|
|
17
17
|
@click.pass_context
|
|
18
|
-
|
|
19
|
-
async def info(ctx: click.Context) -> None:
|
|
18
|
+
def info(ctx: click.Context) -> None:
|
|
20
19
|
"""Get information about the project that is currently being used"""
|
|
21
20
|
ctx_config = ctx.ensure_object(dict)["config"]
|
|
22
21
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
@@ -27,11 +26,11 @@ async def info(ctx: click.Context) -> None:
|
|
|
27
26
|
return
|
|
28
27
|
|
|
29
28
|
click.echo(FeedbackManager.highlight(message="» Tinybird Cloud:"))
|
|
30
|
-
cloud_table, cloud_columns =
|
|
29
|
+
cloud_table, cloud_columns = get_cloud_info(ctx_config)
|
|
31
30
|
click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
|
|
32
|
-
local_table, local_columns =
|
|
31
|
+
local_table, local_columns = get_local_info(ctx_config)
|
|
33
32
|
click.echo(FeedbackManager.highlight(message="\n» Project:"))
|
|
34
|
-
project_table, project_columns =
|
|
33
|
+
project_table, project_columns = get_project_info(project.folder)
|
|
35
34
|
if output == "json":
|
|
36
35
|
cloud_data = {}
|
|
37
36
|
if cloud_columns and cloud_table and isinstance(cloud_table, list) and len(cloud_table) > 0:
|
|
@@ -47,7 +46,7 @@ async def info(ctx: click.Context) -> None:
|
|
|
47
46
|
echo_json({"cloud": cloud_data, "local": local_data, "project": project_data})
|
|
48
47
|
|
|
49
48
|
|
|
50
|
-
|
|
49
|
+
def get_cloud_info(ctx_config: Dict[str, Any]) -> Tuple[Iterable[Any], List[str]]:
|
|
51
50
|
config = CLIConfig.get_project_config()
|
|
52
51
|
|
|
53
52
|
try:
|
|
@@ -57,7 +56,7 @@ async def get_cloud_info(ctx_config: Dict[str, Any]) -> Tuple[Iterable[Any], Lis
|
|
|
57
56
|
ui_host = get_display_cloud_host(api_host)
|
|
58
57
|
user_email = config.get("user_email") or "No user email found"
|
|
59
58
|
user_token = config.get_user_token() or "No user token found"
|
|
60
|
-
return
|
|
59
|
+
return get_env_info(client, ctx_config, user_email, token, user_token, api_host, ui_host)
|
|
61
60
|
except Exception:
|
|
62
61
|
click.echo(
|
|
63
62
|
FeedbackManager.warning(
|
|
@@ -67,16 +66,16 @@ async def get_cloud_info(ctx_config: Dict[str, Any]) -> Tuple[Iterable[Any], Lis
|
|
|
67
66
|
return [], []
|
|
68
67
|
|
|
69
68
|
|
|
70
|
-
|
|
69
|
+
def get_local_info(config: Dict[str, Any]) -> Tuple[Iterable[Any], List[str]]:
|
|
71
70
|
try:
|
|
72
|
-
local_config =
|
|
71
|
+
local_config = get_tinybird_local_config(config, test=False, silent=False)
|
|
73
72
|
local_client = local_config.get_client(host=TB_LOCAL_ADDRESS, staging=False)
|
|
74
73
|
user_email = local_config.get_user_email() or "No user email found"
|
|
75
74
|
token = local_config.get_token() or "No token found"
|
|
76
75
|
user_token = local_config.get_user_token() or "No user token found"
|
|
77
76
|
api_host = TB_LOCAL_ADDRESS
|
|
78
77
|
ui_host = get_display_cloud_host(api_host)
|
|
79
|
-
return
|
|
78
|
+
return get_env_info(local_client, config, user_email, token, user_token, api_host, ui_host, is_local=True)
|
|
80
79
|
except CLILocalException as e:
|
|
81
80
|
raise e
|
|
82
81
|
except Exception as e:
|
|
@@ -88,7 +87,7 @@ async def get_local_info(config: Dict[str, Any]) -> Tuple[Iterable[Any], List[st
|
|
|
88
87
|
return [], []
|
|
89
88
|
|
|
90
89
|
|
|
91
|
-
|
|
90
|
+
def get_env_info(
|
|
92
91
|
client: TinyB,
|
|
93
92
|
config: Dict[str, Any],
|
|
94
93
|
user_email: str,
|
|
@@ -98,8 +97,8 @@ async def get_env_info(
|
|
|
98
97
|
ui_host: str,
|
|
99
98
|
is_local=False,
|
|
100
99
|
) -> Tuple[List[Any], List[str]]:
|
|
101
|
-
user_workspaces =
|
|
102
|
-
current_workspace =
|
|
100
|
+
user_workspaces = client.user_workspaces(version="v1")
|
|
101
|
+
current_workspace = client.workspace_info(version="v1")
|
|
103
102
|
|
|
104
103
|
def _get_current_workspace(user_workspaces: Dict[str, Any], current_workspace_id: str) -> Optional[Dict[str, Any]]:
|
|
105
104
|
def get_workspace_by_name(workspaces: List[Dict[str, Any]], name: str) -> Optional[Dict[str, Any]]:
|
|
@@ -136,7 +135,7 @@ async def get_env_info(
|
|
|
136
135
|
return table, columns
|
|
137
136
|
|
|
138
137
|
|
|
139
|
-
|
|
138
|
+
def get_project_info(project_path: Optional[str] = None) -> Tuple[Iterable[Any], List[str]]:
|
|
140
139
|
config = CLIConfig.get_project_config()
|
|
141
140
|
tinyb_path = config.get_tinyb_file()
|
|
142
141
|
current_path = os.getcwd()
|