tinybird 0.0.1.dev34__py3-none-any.whl → 0.0.1.dev36__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/context.py +1 -1
- tinybird/feedback_manager.py +6 -0
- tinybird/prompts.py +6 -0
- tinybird/sql_toolset.py +9 -2
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +2 -2
- tinybird/tb/modules/build.py +53 -12
- tinybird/tb/modules/cli.py +7 -94
- tinybird/tb/modules/create.py +4 -4
- tinybird/tb/modules/datafile/build.py +2 -2
- tinybird/tb/modules/datafile/common.py +17 -1
- tinybird/tb/modules/datasource.py +3 -471
- tinybird/tb/modules/{deploy.py → deployment.py} +22 -12
- tinybird/tb/modules/endpoint.py +187 -0
- tinybird/tb/modules/llm.py +10 -16
- tinybird/tb/modules/llm_utils.py +87 -0
- tinybird/tb/modules/local.py +4 -1
- tinybird/tb/modules/local_common.py +2 -2
- tinybird/tb/modules/mock.py +3 -4
- tinybird/tb/modules/pipe.py +1 -254
- tinybird/tb/modules/shell.py +8 -1
- tinybird/tb/modules/test.py +2 -2
- tinybird/tb/modules/update.py +4 -4
- tinybird/tb/modules/watch.py +4 -4
- tinybird/tb/modules/workspace.py +0 -96
- tinybird/tb_cli_modules/common.py +19 -17
- {tinybird-0.0.1.dev34.dist-info → tinybird-0.0.1.dev36.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev34.dist-info → tinybird-0.0.1.dev36.dist-info}/RECORD +31 -31
- tinybird/tb/modules/connection.py +0 -803
- {tinybird-0.0.1.dev34.dist-info → tinybird-0.0.1.dev36.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev34.dist-info → tinybird-0.0.1.dev36.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev34.dist-info → tinybird-0.0.1.dev36.dist-info}/top_level.txt +0 -0
|
@@ -5,39 +5,22 @@
|
|
|
5
5
|
|
|
6
6
|
import asyncio
|
|
7
7
|
import json
|
|
8
|
-
import logging
|
|
9
8
|
import re
|
|
10
|
-
from typing import
|
|
9
|
+
from typing import Optional
|
|
11
10
|
|
|
12
11
|
import click
|
|
13
12
|
import humanfriendly
|
|
14
13
|
from click import Context
|
|
15
14
|
|
|
16
|
-
from tinybird.client import AuthNoTokenException,
|
|
17
|
-
from tinybird.config import get_display_host
|
|
18
|
-
from tinybird.tb.modules.config import CLIConfig
|
|
19
|
-
|
|
20
|
-
if TYPE_CHECKING:
|
|
21
|
-
from tinybird.connectors import Connector
|
|
22
|
-
|
|
15
|
+
from tinybird.client import AuthNoTokenException, DoesNotExistException, TinyB
|
|
23
16
|
from tinybird.tb.modules.cli import cli
|
|
24
17
|
from tinybird.tb.modules.common import (
|
|
25
18
|
_analyze,
|
|
26
|
-
_generate_datafile,
|
|
27
|
-
ask_for_user_token,
|
|
28
|
-
autocomplete_topics,
|
|
29
|
-
check_user_token,
|
|
30
19
|
coro,
|
|
31
20
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
32
21
|
get_format_from_filename_or_url,
|
|
33
22
|
load_connector_config,
|
|
34
23
|
push_data,
|
|
35
|
-
sync_data,
|
|
36
|
-
validate_datasource_name,
|
|
37
|
-
validate_kafka_auto_offset_reset,
|
|
38
|
-
validate_kafka_group,
|
|
39
|
-
validate_kafka_topic,
|
|
40
|
-
wait_job,
|
|
41
24
|
)
|
|
42
25
|
from tinybird.tb.modules.datafile.common import get_name_version
|
|
43
26
|
from tinybird.tb.modules.exceptions import CLIDatasourceException
|
|
@@ -66,7 +49,7 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
66
49
|
|
|
67
50
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
68
51
|
ds = await client.datasources()
|
|
69
|
-
columns = ["
|
|
52
|
+
columns = ["shared from", "name", "row_count", "size", "created at", "updated at", "connection"]
|
|
70
53
|
table_human_readable = []
|
|
71
54
|
table_machine_readable = []
|
|
72
55
|
pattern = re.compile(match) if match else None
|
|
@@ -89,7 +72,6 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
89
72
|
|
|
90
73
|
table_human_readable.append(
|
|
91
74
|
(
|
|
92
|
-
tk["version"] if tk["version"] is not None else "",
|
|
93
75
|
shared_from,
|
|
94
76
|
name,
|
|
95
77
|
humanfriendly.format_number(stats.get("row_count")) if stats.get("row_count", None) else "-",
|
|
@@ -101,7 +83,6 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
101
83
|
)
|
|
102
84
|
table_machine_readable.append(
|
|
103
85
|
{
|
|
104
|
-
"version": tk["version"] if tk["version"] is not None else "",
|
|
105
86
|
"shared from": shared_from,
|
|
106
87
|
"name": name,
|
|
107
88
|
"row_count": stats.get("row_count", None) or "-",
|
|
@@ -249,62 +230,6 @@ async def datasource_analyze(ctx, url_or_file, connector):
|
|
|
249
230
|
_table("dialect", ("name", "value"), values)
|
|
250
231
|
|
|
251
232
|
|
|
252
|
-
@datasource.command(name="rm")
|
|
253
|
-
@click.argument("datasource_name")
|
|
254
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
255
|
-
@click.pass_context
|
|
256
|
-
@coro
|
|
257
|
-
async def datasource_delete(ctx: Context, datasource_name: str, yes: bool):
|
|
258
|
-
"""Delete a data source"""
|
|
259
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
260
|
-
try:
|
|
261
|
-
datasource = await client.get_datasource(datasource_name)
|
|
262
|
-
except AuthNoTokenException:
|
|
263
|
-
raise
|
|
264
|
-
except DoesNotExistException:
|
|
265
|
-
raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
|
|
266
|
-
except Exception as e:
|
|
267
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
|
|
268
|
-
connector = datasource.get("service", False)
|
|
269
|
-
|
|
270
|
-
if connector:
|
|
271
|
-
click.echo(FeedbackManager.warning_datasource_is_connected(datasource=datasource_name, connector=connector))
|
|
272
|
-
|
|
273
|
-
try:
|
|
274
|
-
response = await client.datasource_delete(datasource_name, dry_run=True)
|
|
275
|
-
dependencies_information = f'The Data Source is used in => Pipes="{response["dependent_pipes"]}", nodes="{response["dependent_pipes"]}"'
|
|
276
|
-
dependencies_information = (
|
|
277
|
-
dependencies_information if response["dependent_pipes"] else "The Data Source is not used in any Pipe"
|
|
278
|
-
)
|
|
279
|
-
warning_message = f"\nDo you want to delete {datasource_name}? Once deleted, it can't be recovered."
|
|
280
|
-
except CanNotBeDeletedException as e:
|
|
281
|
-
if "downstream" not in str(e):
|
|
282
|
-
dependencies_information = str(e)
|
|
283
|
-
warning_message = f"\nDo you want to unlink and delete {datasource_name}? This action can't be undone."
|
|
284
|
-
else:
|
|
285
|
-
raise CLIDatasourceException(
|
|
286
|
-
FeedbackManager.error_datasource_can_not_be_deleted(datasource=datasource_name, error=e)
|
|
287
|
-
)
|
|
288
|
-
|
|
289
|
-
if yes or click.confirm(
|
|
290
|
-
FeedbackManager.warning_confirm_delete_datasource(
|
|
291
|
-
warning_message=warning_message, dependencies_information=dependencies_information
|
|
292
|
-
)
|
|
293
|
-
):
|
|
294
|
-
try:
|
|
295
|
-
await client.datasource_delete(datasource_name, force=True)
|
|
296
|
-
except DoesNotExistException:
|
|
297
|
-
raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
|
|
298
|
-
except CanNotBeDeletedException as e:
|
|
299
|
-
raise CLIDatasourceException(
|
|
300
|
-
FeedbackManager.error_datasource_can_not_be_deleted(datasource=datasource_name, error=e)
|
|
301
|
-
)
|
|
302
|
-
except Exception as e:
|
|
303
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
|
|
304
|
-
|
|
305
|
-
click.echo(FeedbackManager.success_delete_datasource(datasource=datasource_name))
|
|
306
|
-
|
|
307
|
-
|
|
308
233
|
@datasource.command(name="truncate")
|
|
309
234
|
@click.argument("datasource_name", required=True)
|
|
310
235
|
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
@@ -433,396 +358,3 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
|
|
|
433
358
|
raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
|
|
434
359
|
except Exception as e:
|
|
435
360
|
raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
@datasource.command(
|
|
439
|
-
name="generate",
|
|
440
|
-
short_help="Generates a .datasource file based on a sample CSV, NDJSON or Parquet file from local disk or url",
|
|
441
|
-
)
|
|
442
|
-
@click.argument("filenames", nargs=-1, default=None)
|
|
443
|
-
@click.option("--force", is_flag=True, default=False, help="Override existing files")
|
|
444
|
-
@click.option(
|
|
445
|
-
"--connector",
|
|
446
|
-
type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
|
|
447
|
-
help="Use from one of the selected connectors. In this case pass a table name as a parameter instead of a file name",
|
|
448
|
-
hidden=True,
|
|
449
|
-
)
|
|
450
|
-
@click.pass_context
|
|
451
|
-
@coro
|
|
452
|
-
async def generate_datasource(ctx: Context, connector: str, filenames, force: bool):
|
|
453
|
-
"""Generate a data source file based on a sample CSV file from local disk or url"""
|
|
454
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
455
|
-
|
|
456
|
-
_connector: Optional[Connector] = None
|
|
457
|
-
if connector:
|
|
458
|
-
load_connector_config(ctx, connector, False, check_uninstalled=False)
|
|
459
|
-
if connector not in ctx.ensure_object(dict):
|
|
460
|
-
raise CLIDatasourceException(FeedbackManager.error_connector_not_configured(connector=connector))
|
|
461
|
-
else:
|
|
462
|
-
_connector = ctx.ensure_object(dict)[connector]
|
|
463
|
-
|
|
464
|
-
for filename in filenames:
|
|
465
|
-
await _generate_datafile(
|
|
466
|
-
filename, client, force=force, format=get_format_from_filename_or_url(filename), connector=_connector
|
|
467
|
-
)
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
@datasource.command(name="connect")
|
|
471
|
-
@click.argument("connection")
|
|
472
|
-
@click.argument("datasource_name")
|
|
473
|
-
@click.option("--kafka-topic", "topic", help="For Kafka connections: topic", shell_complete=autocomplete_topics)
|
|
474
|
-
@click.option("--topic", "topic", hidden=True)
|
|
475
|
-
@click.option("--kafka-group", "group", help="For Kafka connections: group ID")
|
|
476
|
-
@click.option("--group", "group", hidden=True)
|
|
477
|
-
@click.option(
|
|
478
|
-
"--kafka-auto-offset-reset",
|
|
479
|
-
"auto_offset_reset",
|
|
480
|
-
default=None,
|
|
481
|
-
type=click.Choice(["latest", "earliest"], case_sensitive=False),
|
|
482
|
-
help='Kafka auto.offset.reset config. Valid values are: ["latest", "earliest"]',
|
|
483
|
-
)
|
|
484
|
-
@click.option("--auto-offset-reset", "auto_offset_reset", hidden=True)
|
|
485
|
-
@click.pass_context
|
|
486
|
-
@coro
|
|
487
|
-
# Example usage: tb datasource connect 776824da-ac64-4de4-b8b8-b909f69d5ed5 new_ds --topic a --group b --auto-offset-reset latest
|
|
488
|
-
async def datasource_connect(ctx, connection, datasource_name, topic, group, auto_offset_reset):
|
|
489
|
-
"""Create a new datasource from an existing connection"""
|
|
490
|
-
|
|
491
|
-
validate_datasource_name(datasource_name)
|
|
492
|
-
|
|
493
|
-
client: TinyB = ctx.obj["client"]
|
|
494
|
-
|
|
495
|
-
connector = await client.get_connector(connection, key="name") or await client.get_connector(connection, key="id")
|
|
496
|
-
if not connector:
|
|
497
|
-
raise CLIDatasourceException(FeedbackManager.error_connection_does_not_exists(connection=connection))
|
|
498
|
-
|
|
499
|
-
service: str = connector.get("service", "")
|
|
500
|
-
if service == "kafka":
|
|
501
|
-
topic and validate_kafka_topic(topic)
|
|
502
|
-
group and validate_kafka_group(group)
|
|
503
|
-
auto_offset_reset and validate_kafka_auto_offset_reset(auto_offset_reset)
|
|
504
|
-
|
|
505
|
-
if not topic:
|
|
506
|
-
try:
|
|
507
|
-
topics = await client.kafka_list_topics(connection)
|
|
508
|
-
click.echo("We've discovered the following topics:")
|
|
509
|
-
for t in topics:
|
|
510
|
-
click.echo(f" {t}")
|
|
511
|
-
except Exception as e:
|
|
512
|
-
logging.debug(f"Error listing topics: {e}")
|
|
513
|
-
topic = click.prompt("Kafka topic")
|
|
514
|
-
validate_kafka_topic(topic)
|
|
515
|
-
if not group:
|
|
516
|
-
group = click.prompt("Kafka group")
|
|
517
|
-
validate_kafka_group(group)
|
|
518
|
-
if not auto_offset_reset:
|
|
519
|
-
click.echo("Kafka doesn't seem to have prior commits on this topic and group ID")
|
|
520
|
-
click.echo("Setting auto.offset.reset is required. Valid values:")
|
|
521
|
-
click.echo(" latest Skip earlier messages and ingest only new messages")
|
|
522
|
-
click.echo(" earliest Start ingestion from the first message")
|
|
523
|
-
auto_offset_reset = click.prompt("Kafka auto.offset.reset config")
|
|
524
|
-
validate_kafka_auto_offset_reset(auto_offset_reset)
|
|
525
|
-
if not click.confirm("Proceed?"):
|
|
526
|
-
return
|
|
527
|
-
resp = await client.datasource_kafka_connect(connection, datasource_name, topic, group, auto_offset_reset)
|
|
528
|
-
datasource_id = resp["datasource"]["id"]
|
|
529
|
-
click.echo(FeedbackManager.success_datasource_kafka_connected(id=datasource_id))
|
|
530
|
-
else:
|
|
531
|
-
raise CLIDatasourceException(FeedbackManager.error_unknown_connection_service(service=service))
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
@datasource.command(name="share")
|
|
535
|
-
@click.argument("datasource_name")
|
|
536
|
-
@click.argument("workspace_name_or_id")
|
|
537
|
-
@click.option("--user_token", default=None, help="When passed, we won't prompt asking for it")
|
|
538
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
539
|
-
@click.pass_context
|
|
540
|
-
@coro
|
|
541
|
-
async def datasource_share(ctx: Context, datasource_name: str, workspace_name_or_id: str, user_token: str, yes: bool):
|
|
542
|
-
"""Share a datasource"""
|
|
543
|
-
|
|
544
|
-
config = CLIConfig.get_project_config()
|
|
545
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
546
|
-
host = config.get_host() or CLIConfig.DEFAULTS["host"]
|
|
547
|
-
ui_host = get_display_host(host)
|
|
548
|
-
|
|
549
|
-
_datasource: Dict[str, Any] = {}
|
|
550
|
-
try:
|
|
551
|
-
_datasource = await client.get_datasource(datasource_name)
|
|
552
|
-
except AuthNoTokenException:
|
|
553
|
-
raise
|
|
554
|
-
except DoesNotExistException:
|
|
555
|
-
raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
|
|
556
|
-
except Exception as e:
|
|
557
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=str(e)))
|
|
558
|
-
|
|
559
|
-
workspaces: List[Dict[str, Any]] = (await client.user_workspaces()).get("workspaces", [])
|
|
560
|
-
destination_workspace = next(
|
|
561
|
-
(
|
|
562
|
-
workspace
|
|
563
|
-
for workspace in workspaces
|
|
564
|
-
if workspace["name"] == workspace_name_or_id or workspace["id"] == workspace_name_or_id
|
|
565
|
-
),
|
|
566
|
-
None,
|
|
567
|
-
)
|
|
568
|
-
current_workspace = next((workspace for workspace in workspaces if workspace["id"] == config["id"]), None)
|
|
569
|
-
|
|
570
|
-
if not destination_workspace:
|
|
571
|
-
raise CLIDatasourceException(FeedbackManager.error_workspace(workspace=workspace_name_or_id))
|
|
572
|
-
|
|
573
|
-
if not current_workspace:
|
|
574
|
-
raise CLIDatasourceException(FeedbackManager.error_not_authenticated())
|
|
575
|
-
|
|
576
|
-
if not user_token:
|
|
577
|
-
user_token = ask_for_user_token("share a Data Source", ui_host)
|
|
578
|
-
await check_user_token(ctx, user_token)
|
|
579
|
-
|
|
580
|
-
client.token = user_token
|
|
581
|
-
|
|
582
|
-
if yes or click.confirm(
|
|
583
|
-
FeedbackManager.warning_datasource_share(
|
|
584
|
-
datasource=datasource_name,
|
|
585
|
-
source_workspace=current_workspace.get("name"),
|
|
586
|
-
destination_workspace=destination_workspace["name"],
|
|
587
|
-
)
|
|
588
|
-
):
|
|
589
|
-
try:
|
|
590
|
-
await client.datasource_share(
|
|
591
|
-
datasource_id=_datasource.get("id", ""),
|
|
592
|
-
current_workspace_id=current_workspace.get("id", ""),
|
|
593
|
-
destination_workspace_id=destination_workspace.get("id", ""),
|
|
594
|
-
)
|
|
595
|
-
click.echo(
|
|
596
|
-
FeedbackManager.success_datasource_shared(
|
|
597
|
-
datasource=datasource_name, workspace=destination_workspace["name"]
|
|
598
|
-
)
|
|
599
|
-
)
|
|
600
|
-
except Exception as e:
|
|
601
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=str(e)))
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
@datasource.command(name="unshare")
|
|
605
|
-
@click.argument("datasource_name")
|
|
606
|
-
@click.argument("workspace_name_or_id")
|
|
607
|
-
@click.option("--user_token", default=None, help="When passed, we won't prompt asking for it")
|
|
608
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
609
|
-
@click.pass_context
|
|
610
|
-
@coro
|
|
611
|
-
async def datasource_unshare(ctx: Context, datasource_name: str, workspace_name_or_id: str, user_token: str, yes: bool):
|
|
612
|
-
"""Unshare a datasource"""
|
|
613
|
-
|
|
614
|
-
config = CLIConfig.get_project_config()
|
|
615
|
-
client = config.get_client()
|
|
616
|
-
host = config.get_host() or CLIConfig.DEFAULTS["host"]
|
|
617
|
-
ui_host = get_display_host(host)
|
|
618
|
-
|
|
619
|
-
_datasource: Dict[str, Any] = {}
|
|
620
|
-
try:
|
|
621
|
-
_datasource = await client.get_datasource(datasource_name)
|
|
622
|
-
except AuthNoTokenException:
|
|
623
|
-
raise
|
|
624
|
-
except DoesNotExistException:
|
|
625
|
-
raise CLIDatasourceException(FeedbackManager.error_datasource_does_not_exist(datasource=datasource_name))
|
|
626
|
-
except Exception as e:
|
|
627
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=str(e)))
|
|
628
|
-
|
|
629
|
-
workspaces: List[Dict[str, Any]] = (await client.user_workspaces()).get("workspaces", [])
|
|
630
|
-
destination_workspace = next(
|
|
631
|
-
(
|
|
632
|
-
workspace
|
|
633
|
-
for workspace in workspaces
|
|
634
|
-
if workspace["name"] == workspace_name_or_id or workspace["id"] == workspace_name_or_id
|
|
635
|
-
),
|
|
636
|
-
None,
|
|
637
|
-
)
|
|
638
|
-
current_workspace = next((workspace for workspace in workspaces if workspace["id"] == config["id"]), None)
|
|
639
|
-
|
|
640
|
-
if not destination_workspace:
|
|
641
|
-
raise CLIDatasourceException(FeedbackManager.error_workspace(workspace=workspace_name_or_id))
|
|
642
|
-
|
|
643
|
-
if not current_workspace:
|
|
644
|
-
raise CLIDatasourceException(FeedbackManager.error_not_authenticated())
|
|
645
|
-
|
|
646
|
-
if not user_token:
|
|
647
|
-
user_token = ask_for_user_token("unshare a Data Source", ui_host)
|
|
648
|
-
await check_user_token(ctx, user_token)
|
|
649
|
-
|
|
650
|
-
client.token = user_token
|
|
651
|
-
|
|
652
|
-
if yes or click.confirm(
|
|
653
|
-
FeedbackManager.warning_datasource_unshare(
|
|
654
|
-
datasource=datasource_name,
|
|
655
|
-
source_workspace=current_workspace.get("name"),
|
|
656
|
-
destination_workspace=destination_workspace["name"],
|
|
657
|
-
)
|
|
658
|
-
):
|
|
659
|
-
try:
|
|
660
|
-
await client.datasource_unshare(
|
|
661
|
-
datasource_id=_datasource.get("id", ""),
|
|
662
|
-
current_workspace_id=current_workspace.get("id", ""),
|
|
663
|
-
destination_workspace_id=destination_workspace.get("id", ""),
|
|
664
|
-
)
|
|
665
|
-
click.echo(
|
|
666
|
-
FeedbackManager.success_datasource_unshared(
|
|
667
|
-
datasource=datasource_name, workspace=destination_workspace["name"]
|
|
668
|
-
)
|
|
669
|
-
)
|
|
670
|
-
except Exception as e:
|
|
671
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=str(e)))
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
@datasource.command(name="sync")
|
|
675
|
-
@click.argument("datasource_name")
|
|
676
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
677
|
-
@click.pass_context
|
|
678
|
-
@coro
|
|
679
|
-
async def datasource_sync(ctx, datasource_name: str, yes: bool):
|
|
680
|
-
"""Sync from connector defined in .datasource file"""
|
|
681
|
-
|
|
682
|
-
try:
|
|
683
|
-
await sync_data(ctx, datasource_name, yes)
|
|
684
|
-
except AuthNoTokenException:
|
|
685
|
-
raise
|
|
686
|
-
except Exception as e:
|
|
687
|
-
raise CLIDatasourceException(FeedbackManager.error_syncing_datasource(datasource=datasource_name, error=str(e)))
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
@datasource.command(name="exchange", hidden=True)
|
|
691
|
-
@click.argument("datasource_a", required=True)
|
|
692
|
-
@click.argument("datasource_b", required=True)
|
|
693
|
-
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
694
|
-
@click.pass_context
|
|
695
|
-
@coro
|
|
696
|
-
async def datasource_exchange(ctx, datasource_a: str, datasource_b: str, yes: bool):
|
|
697
|
-
"""Exchange two data sources"""
|
|
698
|
-
|
|
699
|
-
client = ctx.obj["client"]
|
|
700
|
-
|
|
701
|
-
try:
|
|
702
|
-
if yes or click.confirm(FeedbackManager.warning_exchange(datasource_a=datasource_a, datasource_b=datasource_b)):
|
|
703
|
-
await client.datasource_exchange(datasource_a, datasource_b)
|
|
704
|
-
except Exception as e:
|
|
705
|
-
raise CLIDatasourceException(FeedbackManager.error_exception(error=e))
|
|
706
|
-
|
|
707
|
-
click.echo(FeedbackManager.success_exchange_datasources(datasource_a=datasource_a, datasource_b=datasource_b))
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
@datasource.command(name="copy")
|
|
711
|
-
@click.argument("datasource_name")
|
|
712
|
-
@click.option(
|
|
713
|
-
"--sql",
|
|
714
|
-
default=None,
|
|
715
|
-
help="Freeform SQL query to select what is copied from Main into the Branch Data Source",
|
|
716
|
-
required=False,
|
|
717
|
-
)
|
|
718
|
-
@click.option(
|
|
719
|
-
"--sql-from-main",
|
|
720
|
-
is_flag=True,
|
|
721
|
-
default=False,
|
|
722
|
-
help="SQL query selecting * from the same Data Source in Main",
|
|
723
|
-
required=False,
|
|
724
|
-
)
|
|
725
|
-
@click.option("--wait", is_flag=True, default=False, help="Wait for copy job to finish, disabled by default")
|
|
726
|
-
@click.pass_context
|
|
727
|
-
@coro
|
|
728
|
-
async def datasource_copy_from_main(
|
|
729
|
-
ctx: Context, datasource_name: str, sql: str, sql_from_main: bool, wait: bool
|
|
730
|
-
) -> None:
|
|
731
|
-
"""Copy data source from Main."""
|
|
732
|
-
|
|
733
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
734
|
-
|
|
735
|
-
if sql and sql_from_main:
|
|
736
|
-
click.echo(FeedbackManager.error_exception(error="Use --sql or --sql-from-main but not both"))
|
|
737
|
-
return
|
|
738
|
-
|
|
739
|
-
if not sql and not sql_from_main:
|
|
740
|
-
click.echo(FeedbackManager.error_exception(error="Use --sql or --sql-from-main"))
|
|
741
|
-
return
|
|
742
|
-
|
|
743
|
-
response = await client.datasource_query_copy(
|
|
744
|
-
datasource_name, sql if sql else f"SELECT * FROM main.{datasource_name}"
|
|
745
|
-
)
|
|
746
|
-
if "job" not in response:
|
|
747
|
-
raise Exception(response)
|
|
748
|
-
job_id = response["job"]["job_id"]
|
|
749
|
-
job_url = response["job"]["job_url"]
|
|
750
|
-
if sql:
|
|
751
|
-
click.echo(FeedbackManager.info_copy_with_sql_job_url(sql=sql, datasource_name=datasource_name, url=job_url))
|
|
752
|
-
else:
|
|
753
|
-
click.echo(FeedbackManager.info_copy_from_main_job_url(datasource_name=datasource_name, url=job_url))
|
|
754
|
-
if wait:
|
|
755
|
-
base_msg = "Copy from Main Workspace" if sql_from_main else f"Copy from {sql}"
|
|
756
|
-
await wait_job(client, job_id, job_url, f"{base_msg} to {datasource_name}")
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
@datasource.group(name="scheduling")
|
|
760
|
-
@click.pass_context
|
|
761
|
-
def datasource_scheduling(ctx: Context) -> None:
|
|
762
|
-
"""Data Source scheduling commands."""
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
@datasource_scheduling.command(name="state")
|
|
766
|
-
@click.argument("datasource_name")
|
|
767
|
-
@click.pass_context
|
|
768
|
-
@coro
|
|
769
|
-
async def datasource_scheduling_state(ctx: Context, datasource_name: str) -> None:
|
|
770
|
-
"""Get the scheduling state of a Data Source."""
|
|
771
|
-
client: TinyB = ctx.obj["client"]
|
|
772
|
-
try:
|
|
773
|
-
await client.get_datasource(datasource_name) # Check if datasource exists
|
|
774
|
-
state = await client.datasource_scheduling_state(datasource_name)
|
|
775
|
-
click.echo(FeedbackManager.info_datasource_scheduling_state(datasource=datasource_name, state=state))
|
|
776
|
-
|
|
777
|
-
except AuthNoTokenException:
|
|
778
|
-
raise
|
|
779
|
-
except Exception as e:
|
|
780
|
-
raise CLIDatasourceException(
|
|
781
|
-
FeedbackManager.error_datasource_scheduling_state(datasource=datasource_name, error=e)
|
|
782
|
-
)
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
@datasource_scheduling.command(name="pause")
|
|
786
|
-
@click.argument("datasource_name")
|
|
787
|
-
@click.pass_context
|
|
788
|
-
@coro
|
|
789
|
-
async def datasource_scheduling_pause(ctx: Context, datasource_name: str) -> None:
|
|
790
|
-
"""Pause the scheduling of a Data Source."""
|
|
791
|
-
|
|
792
|
-
click.echo(FeedbackManager.info_datasource_scheduling_pause())
|
|
793
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
794
|
-
|
|
795
|
-
try:
|
|
796
|
-
await client.get_datasource(datasource_name) # Check if datasource exists
|
|
797
|
-
await client.datasource_scheduling_pause(datasource_name)
|
|
798
|
-
click.echo(FeedbackManager.success_datasource_scheduling_paused(datasource=datasource_name))
|
|
799
|
-
|
|
800
|
-
except AuthNoTokenException:
|
|
801
|
-
raise
|
|
802
|
-
except Exception as e:
|
|
803
|
-
raise CLIDatasourceException(
|
|
804
|
-
FeedbackManager.error_pausing_datasource_scheduling(datasource=datasource_name, error=e)
|
|
805
|
-
)
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
@datasource_scheduling.command(name="resume")
|
|
809
|
-
@click.argument("datasource_name")
|
|
810
|
-
@click.pass_context
|
|
811
|
-
@coro
|
|
812
|
-
async def datasource_scheduling_resume(ctx: Context, datasource_name: str) -> None:
|
|
813
|
-
"""Resume the scheduling of a Data Source."""
|
|
814
|
-
|
|
815
|
-
click.echo(FeedbackManager.info_datasource_scheduling_resume())
|
|
816
|
-
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
817
|
-
|
|
818
|
-
try:
|
|
819
|
-
await client.get_datasource(datasource_name) # Check if datasource exists
|
|
820
|
-
await client.datasource_scheduling_resume(datasource_name)
|
|
821
|
-
click.echo(FeedbackManager.success_datasource_scheduling_resumed(datasource=datasource_name))
|
|
822
|
-
|
|
823
|
-
except AuthNoTokenException:
|
|
824
|
-
raise
|
|
825
|
-
except Exception as e:
|
|
826
|
-
raise CLIDatasourceException(
|
|
827
|
-
FeedbackManager.error_resuming_datasource_scheduling(datasource=datasource_name, error=e)
|
|
828
|
-
)
|
|
@@ -2,6 +2,7 @@ import glob
|
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
4
|
import time
|
|
5
|
+
from datetime import datetime
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import List, Optional
|
|
7
8
|
|
|
@@ -105,15 +106,15 @@ def rollback_deployment(host: Optional[str], headers: dict) -> None:
|
|
|
105
106
|
click.echo(FeedbackManager.success(message="Deployment rolled back successfully"))
|
|
106
107
|
|
|
107
108
|
|
|
108
|
-
@cli.group(name="
|
|
109
|
-
def
|
|
109
|
+
@cli.group(name="deployment")
|
|
110
|
+
def deployment_group() -> None:
|
|
110
111
|
"""
|
|
111
|
-
|
|
112
|
+
Deployment commands.
|
|
112
113
|
"""
|
|
113
114
|
pass
|
|
114
115
|
|
|
115
116
|
|
|
116
|
-
@
|
|
117
|
+
@deployment_group.command(name="create")
|
|
117
118
|
@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
|
|
118
119
|
@click.option(
|
|
119
120
|
"--wait/--no-wait",
|
|
@@ -192,6 +193,9 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
|
|
|
192
193
|
deployment = result.get("deployment")
|
|
193
194
|
if deployment.get("status") == "failed":
|
|
194
195
|
click.echo(FeedbackManager.error(message="Deployment failed"))
|
|
196
|
+
if auto:
|
|
197
|
+
click.echo(FeedbackManager.error(message="Rolling back deployment"))
|
|
198
|
+
rollback_deployment(client.host, HEADERS)
|
|
195
199
|
return
|
|
196
200
|
|
|
197
201
|
click.echo(FeedbackManager.success(message="Deployment is ready"))
|
|
@@ -200,10 +204,10 @@ def create(ctx: click.Context, project_path: Path, wait: bool, auto: bool) -> No
|
|
|
200
204
|
promote_deployment(client.host, HEADERS)
|
|
201
205
|
|
|
202
206
|
|
|
203
|
-
@
|
|
207
|
+
@deployment_group.command(name="ls")
|
|
204
208
|
@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
|
|
205
209
|
@click.pass_context
|
|
206
|
-
def
|
|
210
|
+
def deployment_ls(ctx: click.Context, project_path: Path) -> None:
|
|
207
211
|
"""
|
|
208
212
|
List all the deployments you have in the project.
|
|
209
213
|
"""
|
|
@@ -217,20 +221,26 @@ def deploy_list(ctx: click.Context, project_path: Path) -> None:
|
|
|
217
221
|
result = r.json()
|
|
218
222
|
logging.debug(json.dumps(result, indent=2))
|
|
219
223
|
|
|
220
|
-
|
|
224
|
+
status_map = {"data_ready": "Ready", "failed": "Failed"}
|
|
225
|
+
columns = ["ID", "Status", "Created at", "Live"]
|
|
221
226
|
table = []
|
|
222
227
|
for deployment in result.get("deployments"):
|
|
223
228
|
table.append(
|
|
224
|
-
[
|
|
229
|
+
[
|
|
230
|
+
deployment.get("id"),
|
|
231
|
+
status_map.get(deployment.get("status"), "In progress"),
|
|
232
|
+
datetime.fromisoformat(deployment.get("created_at")).strftime("%Y-%m-%d %H:%M:%S"),
|
|
233
|
+
deployment.get("live"),
|
|
234
|
+
]
|
|
225
235
|
)
|
|
226
236
|
|
|
227
237
|
echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
|
|
228
238
|
|
|
229
239
|
|
|
230
|
-
@
|
|
240
|
+
@deployment_group.command(name="promote")
|
|
231
241
|
@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
|
|
232
242
|
@click.pass_context
|
|
233
|
-
def
|
|
243
|
+
def deployment_promote(ctx: click.Context, project_path: Path) -> None:
|
|
234
244
|
"""
|
|
235
245
|
Promote last deploy to ready and remove old one.
|
|
236
246
|
"""
|
|
@@ -242,10 +252,10 @@ def deploy_promote(ctx: click.Context, project_path: Path) -> None:
|
|
|
242
252
|
promote_deployment(client.host, HEADERS)
|
|
243
253
|
|
|
244
254
|
|
|
245
|
-
@
|
|
255
|
+
@deployment_group.command(name="rollback")
|
|
246
256
|
@click.argument("project_path", type=click.Path(exists=True), default=Path.cwd())
|
|
247
257
|
@click.pass_context
|
|
248
|
-
def
|
|
258
|
+
def deployment_rollback(ctx: click.Context, project_path: Path) -> None:
|
|
249
259
|
"""
|
|
250
260
|
Rollback to the previous deployment.
|
|
251
261
|
"""
|