tinybird 0.0.1.dev305__py3-none-any.whl → 0.0.1.dev307__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/context.py +0 -1
- tinybird/prompts.py +1 -0
- tinybird/service_datasources.py +120 -0
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -0
- tinybird/tb/client.py +2 -2
- tinybird/tb/modules/build.py +10 -1
- tinybird/tb/modules/cli.py +33 -9
- tinybird/tb/modules/common.py +137 -5
- tinybird/tb/modules/deployment_common.py +8 -3
- tinybird/tb/modules/environment.py +152 -0
- tinybird/tb/modules/feedback_manager.py +3 -3
- tinybird/tb/modules/local.py +17 -3
- tinybird/tb/modules/local_common.py +199 -6
- {tinybird-0.0.1.dev305.dist-info → tinybird-0.0.1.dev307.dist-info}/METADATA +1 -1
- {tinybird-0.0.1.dev305.dist-info → tinybird-0.0.1.dev307.dist-info}/RECORD +19 -18
- {tinybird-0.0.1.dev305.dist-info → tinybird-0.0.1.dev307.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev305.dist-info → tinybird-0.0.1.dev307.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev305.dist-info → tinybird-0.0.1.dev307.dist-info}/top_level.txt +0 -0
tinybird/context.py
CHANGED
|
@@ -12,7 +12,6 @@ table_id: ContextVar[str] = ContextVar("table_id")
|
|
|
12
12
|
hfi_frequency: ContextVar[float] = ContextVar("hfi_frequency")
|
|
13
13
|
hfi_frequency_gatherer: ContextVar[float] = ContextVar("hfi_frequency_gatherer")
|
|
14
14
|
use_gatherer: ContextVar[bool] = ContextVar("use_gatherer")
|
|
15
|
-
allow_gatherer_fallback: ContextVar[bool] = ContextVar("allow_gatherer_fallback")
|
|
16
15
|
gatherer_allow_s3_backup_on_user_errors: ContextVar[bool] = ContextVar("gatherer_allow_s3_backup_on_user_errors")
|
|
17
16
|
disable_template_security_validation: ContextVar[bool] = ContextVar("disable_template_security_validation")
|
|
18
17
|
origin: ContextVar[str] = ContextVar("origin")
|
tinybird/prompts.py
CHANGED
|
@@ -776,6 +776,7 @@ datasource_instructions = """
|
|
|
776
776
|
- Use always json paths to define the schema. Example: `user_id` String `json:$.user_id`,
|
|
777
777
|
- Array columns are supported with a special syntax. Example: `items` Array(String) `json:$.items[:]`
|
|
778
778
|
- If the datasource is using an S3 or GCS connection, they need to set IMPORT_CONNECTION_NAME, IMPORT_BUCKET_URI and IMPORT_SCHEDULE (GCS @on-demand only, S3 supports @auto too)
|
|
779
|
+
- If the datasource is using a Kafka connection, they need to set KAFKA_CONNECTION_NAME as the name of the .connection file, KAFKA_TOPIC topic_name and KAFKA_GROUP_ID as the group id for the datasource
|
|
779
780
|
- Unless the user asks for them, do not include ENGINE_PARTITION_KEY and ENGINE_PRIMARY_KEY.
|
|
780
781
|
- DateTime64 type without precision is not supported. Use DateTime64(3) instead.
|
|
781
782
|
</datasource_file_instructions>
|
tinybird/service_datasources.py
CHANGED
|
@@ -455,6 +455,79 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
|
|
|
455
455
|
{"name": "feature", "type": "String"},
|
|
456
456
|
],
|
|
457
457
|
},
|
|
458
|
+
{
|
|
459
|
+
"name": "tinybird.llm_usage",
|
|
460
|
+
"description": "LLM usage metrics from Tinybird AI features including token consumption, costs, and model usage for each request in the workspace.",
|
|
461
|
+
"dateColumn": "start_time",
|
|
462
|
+
"engine": {
|
|
463
|
+
"engine": "MergeTree",
|
|
464
|
+
"sorting_key": "workspace_id, start_time, user_email, request_id",
|
|
465
|
+
"partition_key": "toYYYYMM(start_time)",
|
|
466
|
+
},
|
|
467
|
+
"columns": [
|
|
468
|
+
{"name": "start_time", "type": "DateTime"},
|
|
469
|
+
{"name": "end_time", "type": "DateTime"},
|
|
470
|
+
{"name": "organization_id", "type": "String"},
|
|
471
|
+
{"name": "organization_name", "type": "String"},
|
|
472
|
+
{"name": "workspace_id", "type": "String"},
|
|
473
|
+
{"name": "workspace_name", "type": "String"},
|
|
474
|
+
{"name": "user_email", "type": "String"},
|
|
475
|
+
{"name": "request_id", "type": "String"},
|
|
476
|
+
{"name": "prompt_tokens", "type": "UInt32"},
|
|
477
|
+
{"name": "completion_tokens", "type": "UInt32"},
|
|
478
|
+
{"name": "total_tokens", "type": "UInt32"},
|
|
479
|
+
{"name": "duration", "type": "Float32"},
|
|
480
|
+
{"name": "cost", "type": "Float32"},
|
|
481
|
+
{"name": "origin", "type": "String"},
|
|
482
|
+
{"name": "feature", "type": "String"},
|
|
483
|
+
],
|
|
484
|
+
},
|
|
485
|
+
{
|
|
486
|
+
"name": "tinybird.query_metrics",
|
|
487
|
+
"description": "Query stats metrics from your workspace.",
|
|
488
|
+
"dateColumn": "event_time",
|
|
489
|
+
"engine": {
|
|
490
|
+
"engine": "ReplacingMergeTree",
|
|
491
|
+
"sorting_key": "event_time, organization_id, query_id",
|
|
492
|
+
"partition_key": "toStartOfDay(event_time)",
|
|
493
|
+
},
|
|
494
|
+
"columns": [
|
|
495
|
+
{"name": "event_time", "type": "DateTime"},
|
|
496
|
+
{"name": "organization_id", "type": "String"},
|
|
497
|
+
{"name": "workspace_id", "type": "String"},
|
|
498
|
+
{"name": "query", "type": "String"},
|
|
499
|
+
{"name": "query_id", "type": "String"},
|
|
500
|
+
{"name": "query_type", "type": "String"},
|
|
501
|
+
{"name": "query_start_time", "type": "DateTime"},
|
|
502
|
+
{"name": "query_duration_ms", "type": "Int32"},
|
|
503
|
+
{"name": "pipe_id", "type": "String"},
|
|
504
|
+
{"name": "job_id", "type": "String"},
|
|
505
|
+
{"name": "job_kind", "type": "String"},
|
|
506
|
+
{"name": "read_rows", "type": "Int32"},
|
|
507
|
+
{"name": "read_bytes", "type": "Int32"},
|
|
508
|
+
{"name": "written_rows", "type": "Int32"},
|
|
509
|
+
{"name": "written_bytes", "type": "Int32"},
|
|
510
|
+
{"name": "memory_usage", "type": "Int32"},
|
|
511
|
+
{"name": "vcpu_time", "type": "Float32"},
|
|
512
|
+
{"name": "exception_code", "type": "Int32"},
|
|
513
|
+
{"name": "exception", "type": "String"},
|
|
514
|
+
],
|
|
515
|
+
},
|
|
516
|
+
{
|
|
517
|
+
"name": "tinybird.vcpu_time",
|
|
518
|
+
"description": "vCPU time metrics from your workspace.",
|
|
519
|
+
"dateColumn": "minute_slot",
|
|
520
|
+
"engine": {
|
|
521
|
+
"engine": "AggregatingMergeTree",
|
|
522
|
+
"sorting_key": "organization_id, minute_slot",
|
|
523
|
+
"partition_key": "toStartOfDay(minute_slot)",
|
|
524
|
+
},
|
|
525
|
+
"columns": [
|
|
526
|
+
{"name": "minute_slot", "type": "DateTime"},
|
|
527
|
+
{"name": "organization_id", "type": "String"},
|
|
528
|
+
{"name": "vcpu_time", "type": "Float64"},
|
|
529
|
+
],
|
|
530
|
+
},
|
|
458
531
|
]
|
|
459
532
|
|
|
460
533
|
|
|
@@ -935,6 +1008,53 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
|
|
|
935
1008
|
{"name": "feature", "type": "String"},
|
|
936
1009
|
],
|
|
937
1010
|
},
|
|
1011
|
+
{
|
|
1012
|
+
"name": "organization.query_metrics",
|
|
1013
|
+
"description": "Query stats metrics from your workspace.",
|
|
1014
|
+
"dateColumn": "event_time",
|
|
1015
|
+
"engine": {
|
|
1016
|
+
"engine": "ReplacingMergeTree",
|
|
1017
|
+
"sorting_key": "event_time, organization_id, query_id",
|
|
1018
|
+
"partition_key": "toStartOfDay(event_time)",
|
|
1019
|
+
},
|
|
1020
|
+
"columns": [
|
|
1021
|
+
{"name": "event_time", "type": "DateTime"},
|
|
1022
|
+
{"name": "organization_id", "type": "String"},
|
|
1023
|
+
{"name": "workspace_id", "type": "String"},
|
|
1024
|
+
{"name": "query", "type": "String"},
|
|
1025
|
+
{"name": "query_id", "type": "String"},
|
|
1026
|
+
{"name": "query_type", "type": "String"},
|
|
1027
|
+
{"name": "query_start_time", "type": "DateTime"},
|
|
1028
|
+
{"name": "query_duration_ms", "type": "Int32"},
|
|
1029
|
+
{"name": "pipe_id", "type": "String"},
|
|
1030
|
+
{"name": "job_id", "type": "String"},
|
|
1031
|
+
{"name": "job_kind", "type": "String"},
|
|
1032
|
+
{"name": "read_rows", "type": "Int32"},
|
|
1033
|
+
{"name": "read_bytes", "type": "Int32"},
|
|
1034
|
+
{"name": "written_rows", "type": "Int32"},
|
|
1035
|
+
{"name": "written_bytes", "type": "Int32"},
|
|
1036
|
+
{"name": "memory_usage", "type": "Int32"},
|
|
1037
|
+
{"name": "vcpu_time", "type": "Float32"},
|
|
1038
|
+
{"name": "exception_code", "type": "Int32"},
|
|
1039
|
+
{"name": "exception", "type": "String"},
|
|
1040
|
+
],
|
|
1041
|
+
},
|
|
1042
|
+
{
|
|
1043
|
+
"name": "organization.vcpu_time",
|
|
1044
|
+
"description": "vCPU time metrics from your workspace.",
|
|
1045
|
+
"dateColumn": "minute_slot",
|
|
1046
|
+
"engine": {
|
|
1047
|
+
"engine": "AggregatingMergeTree",
|
|
1048
|
+
"sorting_key": "organization_id, minute_slot",
|
|
1049
|
+
"partition_key": "toStartOfDay(minute_slot)",
|
|
1050
|
+
},
|
|
1051
|
+
"columns": [
|
|
1052
|
+
{"name": "minute_slot", "type": "DateTime"},
|
|
1053
|
+
{"name": "organization_id", "type": "String"},
|
|
1054
|
+
{"name": "workspace_id", "type": "String"},
|
|
1055
|
+
{"name": "vcpu_time", "type": "Float64"},
|
|
1056
|
+
],
|
|
1057
|
+
},
|
|
938
1058
|
]
|
|
939
1059
|
|
|
940
1060
|
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev307'
|
|
8
|
+
__revision__ = '060369b'
|
tinybird/tb/cli.py
CHANGED
|
@@ -9,6 +9,7 @@ import tinybird.tb.modules.datasource
|
|
|
9
9
|
import tinybird.tb.modules.deployment
|
|
10
10
|
import tinybird.tb.modules.deprecations
|
|
11
11
|
import tinybird.tb.modules.endpoint
|
|
12
|
+
import tinybird.tb.modules.environment
|
|
12
13
|
import tinybird.tb.modules.info
|
|
13
14
|
import tinybird.tb.modules.infra
|
|
14
15
|
import tinybird.tb.modules.job
|
tinybird/tb/client.py
CHANGED
|
@@ -711,7 +711,7 @@ class TinyB:
|
|
|
711
711
|
return self._req(f"/{version}/user/workspaces/?with_environments=true&only_environments=true")
|
|
712
712
|
|
|
713
713
|
def branches(self):
|
|
714
|
-
return self._req("/
|
|
714
|
+
return self._req("/v1/environments")
|
|
715
715
|
|
|
716
716
|
def releases(self, workspace_id):
|
|
717
717
|
return self._req(f"/v0/workspaces/{workspace_id}/releases")
|
|
@@ -740,7 +740,7 @@ class TinyB:
|
|
|
740
740
|
}
|
|
741
741
|
if ignore_datasources:
|
|
742
742
|
params["ignore_datasources"] = ",".join(ignore_datasources)
|
|
743
|
-
return self._req(f"/
|
|
743
|
+
return self._req(f"/v1/environments?{urlencode(params)}", method="POST", data=b"")
|
|
744
744
|
|
|
745
745
|
def branch_workspace_data(
|
|
746
746
|
self,
|
tinybird/tb/modules/build.py
CHANGED
|
@@ -35,7 +35,9 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
35
35
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
36
36
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
37
37
|
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
38
|
-
|
|
38
|
+
|
|
39
|
+
# TODO: Explain that you can use custom environments too once they are open for everyone
|
|
40
|
+
if obj["env"] == "cloud" and not obj["environment"]:
|
|
39
41
|
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
40
42
|
|
|
41
43
|
if project.has_deeper_level():
|
|
@@ -63,11 +65,18 @@ def build(ctx: click.Context, watch: bool) -> None:
|
|
|
63
65
|
@click.option("--ui", is_flag=True, default=False, help="Connect your local project to Tinybird UI")
|
|
64
66
|
@click.pass_context
|
|
65
67
|
def dev(ctx: click.Context, data_origin: str, ui: bool) -> None:
|
|
68
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
69
|
+
|
|
70
|
+
# TODO: Explain that you can use custom environments too once they are open for everyone
|
|
71
|
+
if obj["env"] == "cloud" and not obj["environment"]:
|
|
72
|
+
raise click.ClickException(FeedbackManager.error_build_only_supported_in_local())
|
|
73
|
+
|
|
66
74
|
if data_origin == "cloud":
|
|
67
75
|
return dev_cloud(ctx)
|
|
68
76
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
69
77
|
tb_client: TinyB = ctx.ensure_object(dict)["client"]
|
|
70
78
|
config: Dict[str, Any] = ctx.ensure_object(dict)["config"]
|
|
79
|
+
|
|
71
80
|
build_status = BuildStatus()
|
|
72
81
|
if ui:
|
|
73
82
|
server_thread = threading.Thread(
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -76,6 +76,7 @@ VERSION = f"{__cli__.__version__} (rev {__cli__.__revision__})"
|
|
|
76
76
|
)
|
|
77
77
|
@click.option("--show-tokens", is_flag=True, default=False, help="Enable the output of tokens.")
|
|
78
78
|
@click.option("--cloud/--local", is_flag=True, default=False, help="Run against cloud or local.")
|
|
79
|
+
@click.option("--environment", help="Run against a custom environment.")
|
|
79
80
|
@click.option("--staging", is_flag=True, default=False, help="Run against a staging deployment.")
|
|
80
81
|
@click.option(
|
|
81
82
|
"--output", type=click.Choice(["human", "json", "csv"], case_sensitive=False), default="human", help="Output format"
|
|
@@ -103,6 +104,7 @@ def cli(
|
|
|
103
104
|
version_warning: bool,
|
|
104
105
|
show_tokens: bool,
|
|
105
106
|
cloud: bool,
|
|
107
|
+
environment: Optional[str],
|
|
106
108
|
staging: bool,
|
|
107
109
|
output: str,
|
|
108
110
|
max_depth: int,
|
|
@@ -110,7 +112,7 @@ def cli(
|
|
|
110
112
|
prompt: Optional[str] = None,
|
|
111
113
|
) -> None:
|
|
112
114
|
"""
|
|
113
|
-
|
|
115
|
+
Run just `tb` to use Tinybird Code to interact with your project.
|
|
114
116
|
"""
|
|
115
117
|
|
|
116
118
|
# We need to unpatch for our tests not to break
|
|
@@ -200,17 +202,33 @@ def cli(
|
|
|
200
202
|
return
|
|
201
203
|
|
|
202
204
|
ctx.ensure_object(dict)["project"] = project
|
|
203
|
-
client = create_ctx_client(
|
|
205
|
+
client = create_ctx_client(
|
|
206
|
+
ctx,
|
|
207
|
+
config,
|
|
208
|
+
cloud or bool(environment),
|
|
209
|
+
staging,
|
|
210
|
+
project=project,
|
|
211
|
+
show_warnings=version_warning,
|
|
212
|
+
environment=environment,
|
|
213
|
+
)
|
|
204
214
|
|
|
205
215
|
if client:
|
|
206
216
|
ctx.ensure_object(dict)["client"] = client
|
|
207
217
|
|
|
208
|
-
ctx.ensure_object(dict)["env"] = get_target_env(cloud)
|
|
218
|
+
ctx.ensure_object(dict)["env"] = get_target_env(cloud, environment)
|
|
219
|
+
ctx.ensure_object(dict)["environment"] = environment
|
|
209
220
|
ctx.ensure_object(dict)["output"] = output
|
|
210
221
|
|
|
211
222
|
is_prompt_mode = prompt is not None
|
|
212
223
|
|
|
213
224
|
if is_agent_mode or is_prompt_mode:
|
|
225
|
+
if any(arg in sys.argv for arg in ["--cloud", "--local"]):
|
|
226
|
+
raise CLIException(
|
|
227
|
+
FeedbackManager.error(
|
|
228
|
+
message="Tinybird Code does not support --cloud or --local flags. It will choose the correct environment based on your prompts."
|
|
229
|
+
)
|
|
230
|
+
)
|
|
231
|
+
|
|
214
232
|
run_agent(config, project, dangerously_skip_permissions, prompt=prompt)
|
|
215
233
|
|
|
216
234
|
|
|
@@ -477,7 +495,13 @@ def __hide_click_output() -> None:
|
|
|
477
495
|
|
|
478
496
|
|
|
479
497
|
def create_ctx_client(
|
|
480
|
-
ctx: Context,
|
|
498
|
+
ctx: Context,
|
|
499
|
+
config: Dict[str, Any],
|
|
500
|
+
cloud: bool,
|
|
501
|
+
staging: bool,
|
|
502
|
+
project: Project,
|
|
503
|
+
show_warnings: bool = True,
|
|
504
|
+
environment: Optional[str] = None,
|
|
481
505
|
):
|
|
482
506
|
commands_without_ctx_client = [
|
|
483
507
|
"auth",
|
|
@@ -499,8 +523,8 @@ def create_ctx_client(
|
|
|
499
523
|
if not command or command in commands_without_ctx_client:
|
|
500
524
|
return None
|
|
501
525
|
|
|
502
|
-
commands_always_cloud = ["infra"]
|
|
503
|
-
commands_always_local = ["
|
|
526
|
+
commands_always_cloud = ["infra", "environment"]
|
|
527
|
+
commands_always_local = ["dev", "create"]
|
|
504
528
|
command_always_test = ["test"]
|
|
505
529
|
|
|
506
530
|
if (
|
|
@@ -523,7 +547,7 @@ def create_ctx_client(
|
|
|
523
547
|
if method and show_warnings:
|
|
524
548
|
click.echo(FeedbackManager.gray(message=f"Authentication method: {method}"))
|
|
525
549
|
|
|
526
|
-
return _get_tb_client(config.get("token", ""), config["host"], staging=staging)
|
|
550
|
+
return _get_tb_client(config.get("token", ""), config["host"], staging=staging, environment=environment)
|
|
527
551
|
local = command in commands_always_local
|
|
528
552
|
test = command in command_always_test
|
|
529
553
|
if show_warnings and not local and command not in commands_always_local and command:
|
|
@@ -531,8 +555,8 @@ def create_ctx_client(
|
|
|
531
555
|
return get_tinybird_local_client(config, test=test, staging=staging)
|
|
532
556
|
|
|
533
557
|
|
|
534
|
-
def get_target_env(cloud: bool) -> str:
|
|
535
|
-
if cloud:
|
|
558
|
+
def get_target_env(cloud: bool, environment: Optional[str]) -> str:
|
|
559
|
+
if cloud or bool(environment):
|
|
536
560
|
return "cloud"
|
|
537
561
|
return "local"
|
|
538
562
|
|
tinybird/tb/modules/common.py
CHANGED
|
@@ -378,9 +378,33 @@ def getenv_bool(key: str, default: bool) -> bool:
|
|
|
378
378
|
return v.lower() == "true" or v == "1"
|
|
379
379
|
|
|
380
380
|
|
|
381
|
-
def _get_tb_client(token: str, host: str, staging: bool = False) -> TinyB:
|
|
381
|
+
def _get_tb_client(token: str, host: str, staging: bool = False, environment: Optional[str] = None) -> TinyB:
|
|
382
382
|
disable_ssl: bool = getenv_bool("TB_DISABLE_SSL_CHECKS", False)
|
|
383
|
-
|
|
383
|
+
cloud_client = TinyB(
|
|
384
|
+
token,
|
|
385
|
+
host,
|
|
386
|
+
version=VERSION,
|
|
387
|
+
disable_ssl_checks=disable_ssl,
|
|
388
|
+
send_telemetry=True,
|
|
389
|
+
staging=staging,
|
|
390
|
+
)
|
|
391
|
+
|
|
392
|
+
if not environment:
|
|
393
|
+
return cloud_client
|
|
394
|
+
|
|
395
|
+
workspaces = cloud_client.user_workspaces_and_branches(version="v1")
|
|
396
|
+
workspace = next((w for w in workspaces.get("workspaces", []) if w.get("name") == environment), None)
|
|
397
|
+
if not workspace:
|
|
398
|
+
raise CLIException(FeedbackManager.error_exception(error=f"Environment {environment} not found"))
|
|
399
|
+
|
|
400
|
+
return TinyB(
|
|
401
|
+
workspace.get("token", ""),
|
|
402
|
+
host,
|
|
403
|
+
version=VERSION,
|
|
404
|
+
disable_ssl_checks=disable_ssl,
|
|
405
|
+
send_telemetry=True,
|
|
406
|
+
staging=staging,
|
|
407
|
+
)
|
|
384
408
|
|
|
385
409
|
|
|
386
410
|
def create_tb_client(ctx: Context) -> TinyB:
|
|
@@ -1165,10 +1189,26 @@ def _get_setting_value(connection, setting, sensitive_settings):
|
|
|
1165
1189
|
return connection.get(setting, "")
|
|
1166
1190
|
|
|
1167
1191
|
|
|
1168
|
-
def
|
|
1192
|
+
def get_current_workspace_branches(config: CLIConfig) -> List[Dict[str, Any]]:
|
|
1193
|
+
current_main_workspace: Optional[Dict[str, Any]] = get_current_main_workspace(config)
|
|
1194
|
+
if not current_main_workspace:
|
|
1195
|
+
raise CLIException(FeedbackManager.error_unable_to_identify_main_workspace())
|
|
1196
|
+
|
|
1197
|
+
client = config.get_client()
|
|
1198
|
+
user_branches: List[Dict[str, Any]] = (client.user_workspace_branches("v1")).get("workspaces", [])
|
|
1199
|
+
all_branches: List[Dict[str, Any]] = (client.branches()).get("environments", [])
|
|
1200
|
+
branches = user_branches + [branch for branch in all_branches if branch not in user_branches]
|
|
1201
|
+
|
|
1202
|
+
return [branch for branch in branches if branch.get("main") == current_main_workspace["id"]]
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def switch_workspace(config: CLIConfig, workspace_name_or_id: str, only_environments: bool = False) -> None:
|
|
1169
1206
|
try:
|
|
1170
|
-
|
|
1171
|
-
|
|
1207
|
+
if only_environments:
|
|
1208
|
+
workspaces = get_current_workspace_branches(config)
|
|
1209
|
+
else:
|
|
1210
|
+
response = config.get_client().user_workspaces(version="v1")
|
|
1211
|
+
workspaces = response["workspaces"]
|
|
1172
1212
|
|
|
1173
1213
|
workspace = next(
|
|
1174
1214
|
(
|
|
@@ -2261,3 +2301,95 @@ def update_cli() -> None:
|
|
|
2261
2301
|
click.echo(FeedbackManager.success(message="✓ Tinybird CLI updated"))
|
|
2262
2302
|
else:
|
|
2263
2303
|
click.echo(FeedbackManager.info(message="✓ Tinybird CLI is already up-to-date"))
|
|
2304
|
+
|
|
2305
|
+
|
|
2306
|
+
def create_workspace_branch(
|
|
2307
|
+
branch_name: Optional[str],
|
|
2308
|
+
last_partition: bool,
|
|
2309
|
+
all: bool,
|
|
2310
|
+
ignore_datasources: Optional[List[str]],
|
|
2311
|
+
wait: Optional[bool],
|
|
2312
|
+
) -> None:
|
|
2313
|
+
"""
|
|
2314
|
+
Creates a workspace branch
|
|
2315
|
+
"""
|
|
2316
|
+
config = CLIConfig.get_project_config()
|
|
2317
|
+
_ = try_update_config_with_remote(config)
|
|
2318
|
+
|
|
2319
|
+
try:
|
|
2320
|
+
workspace = get_current_workspace(config)
|
|
2321
|
+
if not workspace:
|
|
2322
|
+
raise CLIWorkspaceException(FeedbackManager.error_workspace())
|
|
2323
|
+
|
|
2324
|
+
if not branch_name:
|
|
2325
|
+
click.echo(FeedbackManager.info_workspace_branch_create_greeting())
|
|
2326
|
+
default_name = f"{workspace['name']}_{uuid.uuid4().hex[0:4]}"
|
|
2327
|
+
branch_name = click.prompt("\Branch name", default=default_name, err=True, type=str)
|
|
2328
|
+
assert isinstance(branch_name, str)
|
|
2329
|
+
|
|
2330
|
+
response = config.get_client().create_workspace_branch(
|
|
2331
|
+
branch_name,
|
|
2332
|
+
last_partition,
|
|
2333
|
+
all,
|
|
2334
|
+
ignore_datasources,
|
|
2335
|
+
)
|
|
2336
|
+
assert isinstance(response, dict)
|
|
2337
|
+
|
|
2338
|
+
is_job: bool = "job" in response
|
|
2339
|
+
is_summary: bool = "partitions" in response
|
|
2340
|
+
|
|
2341
|
+
if not is_job and not is_summary:
|
|
2342
|
+
raise CLIException(str(response))
|
|
2343
|
+
|
|
2344
|
+
if all and not is_job:
|
|
2345
|
+
raise CLIException(str(response))
|
|
2346
|
+
|
|
2347
|
+
click.echo(
|
|
2348
|
+
FeedbackManager.success_workspace_branch_created(workspace_name=workspace["name"], branch_name=branch_name)
|
|
2349
|
+
)
|
|
2350
|
+
|
|
2351
|
+
job_id: Optional[str] = None
|
|
2352
|
+
|
|
2353
|
+
if is_job:
|
|
2354
|
+
job_id = response["job"]["job_id"]
|
|
2355
|
+
job_url = response["job"]["job_url"]
|
|
2356
|
+
click.echo(FeedbackManager.info_data_branch_job_url(url=job_url))
|
|
2357
|
+
|
|
2358
|
+
if wait and is_job:
|
|
2359
|
+
assert isinstance(job_id, str)
|
|
2360
|
+
|
|
2361
|
+
# Await the job to finish and get the result dict
|
|
2362
|
+
job_response = wait_job(config.get_client(), job_id, job_url, "Environment creation")
|
|
2363
|
+
if job_response is None:
|
|
2364
|
+
raise CLIException(f"Empty job API response (job_id: {job_id}, job_url: {job_url})")
|
|
2365
|
+
else:
|
|
2366
|
+
response = job_response.get("result", {})
|
|
2367
|
+
is_summary = "partitions" in response
|
|
2368
|
+
|
|
2369
|
+
except Exception as e:
|
|
2370
|
+
raise CLIException(FeedbackManager.error_exception(error=str(e)))
|
|
2371
|
+
|
|
2372
|
+
|
|
2373
|
+
async def print_current_branch(config: CLIConfig) -> None:
|
|
2374
|
+
_ = try_update_config_with_remote(config, only_if_needed=True)
|
|
2375
|
+
|
|
2376
|
+
response = config.get_client().user_workspaces_and_branches("v1")
|
|
2377
|
+
|
|
2378
|
+
columns = ["name", "id", "workspace"]
|
|
2379
|
+
table = []
|
|
2380
|
+
|
|
2381
|
+
for workspace in response["workspaces"]:
|
|
2382
|
+
if config["id"] == workspace["id"]:
|
|
2383
|
+
click.echo(FeedbackManager.info_current_branch())
|
|
2384
|
+
if workspace.get("is_branch"):
|
|
2385
|
+
name = workspace["name"]
|
|
2386
|
+
main_workspace = get_current_main_workspace(config)
|
|
2387
|
+
assert isinstance(main_workspace, dict)
|
|
2388
|
+
main_name = main_workspace["name"]
|
|
2389
|
+
else:
|
|
2390
|
+
name = MAIN_BRANCH
|
|
2391
|
+
main_name = workspace["name"]
|
|
2392
|
+
table.append([name, workspace["id"], main_name])
|
|
2393
|
+
break
|
|
2394
|
+
|
|
2395
|
+
echo_safe_humanfriendly_tables_format_smart_table(table, column_names=columns)
|
|
@@ -59,9 +59,14 @@ def api_post(
|
|
|
59
59
|
params: Optional[dict] = None,
|
|
60
60
|
) -> dict:
|
|
61
61
|
r = requests.post(url, headers=headers, files=files, params=params)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
try:
|
|
63
|
+
if r.status_code < 300:
|
|
64
|
+
logging.debug(json.dumps(r.json(), indent=2))
|
|
65
|
+
return r.json()
|
|
66
|
+
except json.JSONDecodeError:
|
|
67
|
+
message = "Error parsing response from API"
|
|
68
|
+
click.echo(FeedbackManager.error(message=message))
|
|
69
|
+
sys_exit("deployment_error", message)
|
|
65
70
|
|
|
66
71
|
# Try to parse and print the error from the response
|
|
67
72
|
try:
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# This is a command file for our CLI. Please keep it clean.
|
|
2
|
+
#
|
|
3
|
+
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
|
+
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
|
+
|
|
6
|
+
from typing import List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
import click
|
|
9
|
+
|
|
10
|
+
from tinybird.tb.modules.cli import cli
|
|
11
|
+
from tinybird.tb.modules.common import (
|
|
12
|
+
MAIN_BRANCH,
|
|
13
|
+
create_workspace_branch,
|
|
14
|
+
echo_safe_humanfriendly_tables_format_smart_table,
|
|
15
|
+
get_current_main_workspace,
|
|
16
|
+
get_current_workspace_branches,
|
|
17
|
+
get_workspace_member_email,
|
|
18
|
+
switch_to_workspace_by_user_workspace_data,
|
|
19
|
+
try_update_config_with_remote,
|
|
20
|
+
)
|
|
21
|
+
from tinybird.tb.modules.config import CLIConfig
|
|
22
|
+
from tinybird.tb.modules.exceptions import CLIBranchException, CLIException
|
|
23
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@cli.group(hidden=True)
|
|
27
|
+
def environment() -> None:
|
|
28
|
+
"""Environment commands. Custom environments is an experimental feature in beta."""
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@environment.command(name="ls")
|
|
33
|
+
@click.option("--sort/--no-sort", default=False, help="Sort the table rows by name")
|
|
34
|
+
def branch_ls(sort: bool) -> None:
|
|
35
|
+
"""List all the environments available in the current workspace"""
|
|
36
|
+
|
|
37
|
+
config = CLIConfig.get_project_config()
|
|
38
|
+
_ = try_update_config_with_remote(config, only_if_needed=True)
|
|
39
|
+
|
|
40
|
+
client = config.get_client()
|
|
41
|
+
|
|
42
|
+
current_main_workspace = get_current_main_workspace(config)
|
|
43
|
+
assert isinstance(current_main_workspace, dict)
|
|
44
|
+
|
|
45
|
+
if current_main_workspace["id"] != config["id"]:
|
|
46
|
+
client = config.get_client(token=current_main_workspace["token"])
|
|
47
|
+
|
|
48
|
+
response = client.branches()
|
|
49
|
+
|
|
50
|
+
columns = ["name", "id", "created_at", "owner", "current"]
|
|
51
|
+
|
|
52
|
+
table: List[Tuple[str, str, str, str, bool]] = []
|
|
53
|
+
|
|
54
|
+
for branch in response["environments"]:
|
|
55
|
+
branch_owner_email = get_workspace_member_email(branch, branch["owner"])
|
|
56
|
+
|
|
57
|
+
table.append(
|
|
58
|
+
(branch["name"], branch["id"], branch["created_at"], branch_owner_email, config["id"] == branch["id"])
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
current_branch = [row for row in table if row[4]]
|
|
62
|
+
other_branches = [row for row in table if not row[4]]
|
|
63
|
+
|
|
64
|
+
if sort:
|
|
65
|
+
other_branches.sort(key=lambda x: x[0])
|
|
66
|
+
|
|
67
|
+
sorted_table = current_branch + other_branches
|
|
68
|
+
|
|
69
|
+
click.echo(FeedbackManager.info(message="\n** Environments:"))
|
|
70
|
+
echo_safe_humanfriendly_tables_format_smart_table(sorted_table, column_names=columns)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@environment.command(name="create", short_help="Create a new environment in the current Workspace")
|
|
74
|
+
@click.argument("environment_name", required=False)
|
|
75
|
+
@click.option(
|
|
76
|
+
"--last-partition",
|
|
77
|
+
is_flag=True,
|
|
78
|
+
default=False,
|
|
79
|
+
help="Attach the last modified partition from the current workspace to the new environment",
|
|
80
|
+
)
|
|
81
|
+
@click.option(
|
|
82
|
+
"-i",
|
|
83
|
+
"--ignore-datasource",
|
|
84
|
+
"ignore_datasources",
|
|
85
|
+
type=str,
|
|
86
|
+
multiple=True,
|
|
87
|
+
help="Ignore specified data source partitions",
|
|
88
|
+
)
|
|
89
|
+
@click.option(
|
|
90
|
+
"--wait/--no-wait",
|
|
91
|
+
is_flag=True,
|
|
92
|
+
default=True,
|
|
93
|
+
help="Wait for data branch jobs to finish, showing a progress bar. Disabled by default.",
|
|
94
|
+
)
|
|
95
|
+
def create_environment(
|
|
96
|
+
environment_name: Optional[str], last_partition: bool, ignore_datasources: List[str], wait: bool
|
|
97
|
+
) -> None:
|
|
98
|
+
create_workspace_branch(environment_name, last_partition, False, list(ignore_datasources), wait)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@environment.command(name="rm", short_help="Removes an environment from the workspace. It can't be recovered.")
|
|
102
|
+
@click.argument("branch_name_or_id")
|
|
103
|
+
@click.option("--yes", is_flag=True, default=False, help="Do not ask for confirmation")
|
|
104
|
+
def delete_environment(branch_name_or_id: str, yes: bool) -> None:
|
|
105
|
+
"""Remove an environment"""
|
|
106
|
+
|
|
107
|
+
config = CLIConfig.get_project_config()
|
|
108
|
+
_ = try_update_config_with_remote(config)
|
|
109
|
+
|
|
110
|
+
client = config.get_client()
|
|
111
|
+
|
|
112
|
+
if branch_name_or_id == MAIN_BRANCH:
|
|
113
|
+
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
114
|
+
|
|
115
|
+
try:
|
|
116
|
+
workspace_branches = get_current_workspace_branches(config)
|
|
117
|
+
workspace_to_delete = next(
|
|
118
|
+
(
|
|
119
|
+
workspace
|
|
120
|
+
for workspace in workspace_branches
|
|
121
|
+
if workspace["name"] == branch_name_or_id or workspace["id"] == branch_name_or_id
|
|
122
|
+
),
|
|
123
|
+
None,
|
|
124
|
+
)
|
|
125
|
+
except Exception as e:
|
|
126
|
+
raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
|
|
127
|
+
|
|
128
|
+
if not workspace_to_delete:
|
|
129
|
+
raise CLIBranchException(FeedbackManager.error_branch(branch=branch_name_or_id))
|
|
130
|
+
|
|
131
|
+
if yes or click.confirm(FeedbackManager.warning_confirm_delete_branch(branch=workspace_to_delete["name"])):
|
|
132
|
+
need_to_switch_to_main = workspace_to_delete.get("main") and config["id"] == workspace_to_delete["id"]
|
|
133
|
+
# get origin workspace if deleting current branch
|
|
134
|
+
if need_to_switch_to_main:
|
|
135
|
+
try:
|
|
136
|
+
workspaces = (client.user_workspaces()).get("workspaces", [])
|
|
137
|
+
workspace_main = next(
|
|
138
|
+
(workspace for workspace in workspaces if workspace["id"] == workspace_to_delete["main"]), None
|
|
139
|
+
)
|
|
140
|
+
except Exception:
|
|
141
|
+
workspace_main = None
|
|
142
|
+
try:
|
|
143
|
+
client.delete_branch(workspace_to_delete["id"])
|
|
144
|
+
click.echo(FeedbackManager.success_branch_deleted(branch_name=workspace_to_delete["name"]))
|
|
145
|
+
except Exception as e:
|
|
146
|
+
raise CLIBranchException(FeedbackManager.error_exception(error=str(e)))
|
|
147
|
+
else:
|
|
148
|
+
if need_to_switch_to_main:
|
|
149
|
+
if workspace_main:
|
|
150
|
+
switch_to_workspace_by_user_workspace_data(config, workspace_main)
|
|
151
|
+
else:
|
|
152
|
+
raise CLIException(FeedbackManager.error_switching_to_main())
|
|
@@ -841,7 +841,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
|
|
|
841
841
|
)
|
|
842
842
|
info_populate_job_result = info_message("** Populating job result\n {result}")
|
|
843
843
|
info_populate_job_url = info_message("** Populating job url {url}")
|
|
844
|
-
info_data_branch_job_url = info_message("**
|
|
844
|
+
info_data_branch_job_url = info_message("** Environment job url {url}")
|
|
845
845
|
info_regression_tests_branch_job_url = info_message("** Branch regression tests job url {url}")
|
|
846
846
|
info_merge_branch_job_url = info_message("** Merge Branch deployment job url {url}")
|
|
847
847
|
info_copy_from_main_job_url = info_message("** Copy from 'main' Workspace to '{datasource_name}' job url {url}")
|
|
@@ -1161,7 +1161,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
|
|
|
1161
1161
|
"** Workspace '{workspace_name}' has been created in Organization '{organization_name}' (id: '{organization_id}')"
|
|
1162
1162
|
)
|
|
1163
1163
|
success_workspace_branch_created = success_message(
|
|
1164
|
-
"**
|
|
1164
|
+
"** Environment '{branch_name}' from '{workspace_name}' has been created"
|
|
1165
1165
|
)
|
|
1166
1166
|
success_workspace_data_branch = success_message(
|
|
1167
1167
|
"** Partitions from 'main' Workspace have been attached to the Branch"
|
|
@@ -1174,7 +1174,7 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
|
|
|
1174
1174
|
"Deploying your new '{workspace_name}' workspace, using the '{template}' template:"
|
|
1175
1175
|
)
|
|
1176
1176
|
success_workspace_deleted = success_message("** Workspace '{workspace_name}' deleted")
|
|
1177
|
-
success_branch_deleted = success_message("**
|
|
1177
|
+
success_branch_deleted = success_message("** Environment '{branch_name}' deleted")
|
|
1178
1178
|
success_workspace_user_added = success_message("** User {user} added to workspace '{workspace_name}'")
|
|
1179
1179
|
success_workspace_users_added = success_message("** Users added to workspace '{workspace_name}'")
|
|
1180
1180
|
success_workspace_user_removed = success_message("** User {user} removed from workspace '{workspace_name}'")
|
tinybird/tb/modules/local.py
CHANGED
|
@@ -141,8 +141,19 @@ def remove() -> None:
|
|
|
141
141
|
envvar="TB_LOCAL_WORKSPACE_TOKEN",
|
|
142
142
|
help="Workspace token to use for the Tinybird Local container.",
|
|
143
143
|
)
|
|
144
|
+
@click.option(
|
|
145
|
+
"--watch",
|
|
146
|
+
default=False,
|
|
147
|
+
is_flag=True,
|
|
148
|
+
help="Watch container in the console. Press Ctrl+C to stop the container.",
|
|
149
|
+
)
|
|
144
150
|
def start(
|
|
145
|
-
use_aws_creds: bool,
|
|
151
|
+
use_aws_creds: bool,
|
|
152
|
+
volumes_path: str,
|
|
153
|
+
skip_new_version: bool,
|
|
154
|
+
user_token: str,
|
|
155
|
+
workspace_token: str,
|
|
156
|
+
watch: bool,
|
|
146
157
|
) -> None:
|
|
147
158
|
"""Start Tinybird Local"""
|
|
148
159
|
if volumes_path is not None:
|
|
@@ -152,8 +163,11 @@ def start(
|
|
|
152
163
|
|
|
153
164
|
click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
|
|
154
165
|
docker_client = get_docker_client()
|
|
155
|
-
start_tinybird_local(
|
|
156
|
-
|
|
166
|
+
start_tinybird_local(
|
|
167
|
+
docker_client, use_aws_creds, volumes_path, skip_new_version, user_token, workspace_token, watch=watch
|
|
168
|
+
)
|
|
169
|
+
if not watch:
|
|
170
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
157
171
|
|
|
158
172
|
|
|
159
173
|
@local.command()
|
|
@@ -4,6 +4,7 @@ import logging
|
|
|
4
4
|
import os
|
|
5
5
|
import re
|
|
6
6
|
import subprocess
|
|
7
|
+
import threading
|
|
7
8
|
import time
|
|
8
9
|
import uuid
|
|
9
10
|
from typing import Any, Dict, Optional
|
|
@@ -35,11 +36,18 @@ def get_tinybird_local_client(
|
|
|
35
36
|
config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
|
|
36
37
|
) -> TinyB:
|
|
37
38
|
"""Get a Tinybird client connected to the local environment."""
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
try:
|
|
40
|
+
config = get_tinybird_local_config(config_obj, test=test, silent=silent)
|
|
41
|
+
client = config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
|
|
42
|
+
load_secrets(config_obj.get("path", ""), client)
|
|
43
|
+
return client
|
|
44
|
+
# if some of the API calls to tinybird local fail due to a JSONDecodeError, it means that container is running but it's unhealthy
|
|
45
|
+
except json.JSONDecodeError:
|
|
46
|
+
raise CLILocalException(
|
|
47
|
+
message=FeedbackManager.error(
|
|
48
|
+
message="Tinybird Local is running but it's unhealthy. Please check if it's running and try again. If the problem persists, please run `tb local restart` and try again."
|
|
49
|
+
)
|
|
50
|
+
)
|
|
43
51
|
|
|
44
52
|
|
|
45
53
|
def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
|
|
@@ -140,6 +148,7 @@ def get_local_tokens() -> Dict[str, str]:
|
|
|
140
148
|
},
|
|
141
149
|
)
|
|
142
150
|
|
|
151
|
+
# TODO: If docker errors persist, explain that you can use custom environments too once they are open for everyone
|
|
143
152
|
if container and container.status == "running":
|
|
144
153
|
if container.health == "healthy":
|
|
145
154
|
raise CLILocalException(
|
|
@@ -237,6 +246,7 @@ def start_tinybird_local(
|
|
|
237
246
|
skip_new_version: bool = True,
|
|
238
247
|
user_token: Optional[str] = None,
|
|
239
248
|
workspace_token: Optional[str] = None,
|
|
249
|
+
watch: bool = False,
|
|
240
250
|
) -> None:
|
|
241
251
|
"""Start the Tinybird container."""
|
|
242
252
|
pull_show_prompt = False
|
|
@@ -299,6 +309,190 @@ def start_tinybird_local(
|
|
|
299
309
|
)
|
|
300
310
|
|
|
301
311
|
click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
|
|
312
|
+
|
|
313
|
+
if watch:
|
|
314
|
+
# Stream logs in a separate thread while monitoring container health
|
|
315
|
+
container_ready = threading.Event()
|
|
316
|
+
stop_requested = threading.Event()
|
|
317
|
+
health_check_error = {"message": ""} # Mutable dict to store error message
|
|
318
|
+
|
|
319
|
+
def check_endpoints_health() -> None:
|
|
320
|
+
"""Continuously check /tokens and /v0/health endpoints"""
|
|
321
|
+
# Wait for container to be ready before starting health checks
|
|
322
|
+
container_ready.wait()
|
|
323
|
+
|
|
324
|
+
# Give container a moment to fully start up
|
|
325
|
+
time.sleep(2)
|
|
326
|
+
|
|
327
|
+
check_interval = 10 # Check every 10 seconds
|
|
328
|
+
|
|
329
|
+
while not stop_requested.is_set():
|
|
330
|
+
try:
|
|
331
|
+
# Check /tokens endpoint
|
|
332
|
+
tokens_response = requests.get(f"{TB_LOCAL_ADDRESS}/tokens", timeout=5)
|
|
333
|
+
if tokens_response.status_code != 200:
|
|
334
|
+
health_check_error["message"] = (
|
|
335
|
+
f"/tokens endpoint returned status {tokens_response.status_code}. "
|
|
336
|
+
"Tinybird Local may be unhealthy."
|
|
337
|
+
)
|
|
338
|
+
stop_requested.set()
|
|
339
|
+
break
|
|
340
|
+
|
|
341
|
+
# Check /v0/health endpoint
|
|
342
|
+
health_response = requests.get(f"{TB_LOCAL_ADDRESS}/v0/health", timeout=5)
|
|
343
|
+
if health_response.status_code != 200:
|
|
344
|
+
health_check_error["message"] = (
|
|
345
|
+
f"/v0/health endpoint returned status {health_response.status_code}. "
|
|
346
|
+
"Tinybird Local may be unhealthy."
|
|
347
|
+
)
|
|
348
|
+
stop_requested.set()
|
|
349
|
+
break
|
|
350
|
+
|
|
351
|
+
# Verify tokens response has expected structure
|
|
352
|
+
try:
|
|
353
|
+
tokens_data = tokens_response.json()
|
|
354
|
+
if not all(
|
|
355
|
+
key in tokens_data for key in ["user_token", "admin_token", "workspace_admin_token"]
|
|
356
|
+
):
|
|
357
|
+
health_check_error["message"] = (
|
|
358
|
+
"/tokens endpoint returned unexpected data. Tinybird Local may be unhealthy."
|
|
359
|
+
)
|
|
360
|
+
stop_requested.set()
|
|
361
|
+
break
|
|
362
|
+
except json.JSONDecodeError:
|
|
363
|
+
health_check_error["message"] = (
|
|
364
|
+
"/tokens endpoint returned invalid JSON. Tinybird Local may be unhealthy."
|
|
365
|
+
)
|
|
366
|
+
stop_requested.set()
|
|
367
|
+
break
|
|
368
|
+
|
|
369
|
+
except Exception as e:
|
|
370
|
+
# Check if it's a connection error
|
|
371
|
+
error_str = str(e)
|
|
372
|
+
if "connect" in error_str.lower() or "timeout" in error_str.lower():
|
|
373
|
+
health_check_error["message"] = f"Failed to connect to Tinybird Local: {error_str}"
|
|
374
|
+
else:
|
|
375
|
+
health_check_error["message"] = f"Health check failed: {error_str}"
|
|
376
|
+
stop_requested.set()
|
|
377
|
+
break
|
|
378
|
+
|
|
379
|
+
# Wait before next check
|
|
380
|
+
for _ in range(check_interval):
|
|
381
|
+
if stop_requested.is_set():
|
|
382
|
+
break
|
|
383
|
+
time.sleep(1)
|
|
384
|
+
|
|
385
|
+
def stream_logs_with_health_check() -> None:
|
|
386
|
+
"""Stream logs and monitor container health in parallel"""
|
|
387
|
+
log_names = {
|
|
388
|
+
"/var/log/tinybird-local-setup.log": "SETUP",
|
|
389
|
+
"/var/log/tinybird-local-server.log": "SERVER",
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
# Wait briefly for log files to be created
|
|
393
|
+
retry_count = 0
|
|
394
|
+
max_retries = 3
|
|
395
|
+
exec_result = None
|
|
396
|
+
|
|
397
|
+
while retry_count < max_retries and not stop_requested.is_set():
|
|
398
|
+
try:
|
|
399
|
+
# Try to tail the log files (only new logs, not historical)
|
|
400
|
+
# Use -F to follow by name and retry if files don't exist yet
|
|
401
|
+
cmd = ["tail", "-n", "0", "-F", *log_names.keys()]
|
|
402
|
+
exec_result = container.exec_run(cmd=cmd, stream=True, tty=False, stdout=True, stderr=True)
|
|
403
|
+
break # Success, exit retry loop
|
|
404
|
+
except Exception:
|
|
405
|
+
# Log file might not exist yet, wait and retry
|
|
406
|
+
retry_count += 1
|
|
407
|
+
if retry_count < max_retries:
|
|
408
|
+
time.sleep(2)
|
|
409
|
+
|
|
410
|
+
# If we couldn't start tailing, fall back to container logs
|
|
411
|
+
if not exec_result and not stop_requested.is_set():
|
|
412
|
+
try:
|
|
413
|
+
for line in container.logs(stream=True, follow=True):
|
|
414
|
+
if stop_requested.is_set():
|
|
415
|
+
break
|
|
416
|
+
click.echo(line.decode("utf-8").rstrip())
|
|
417
|
+
return
|
|
418
|
+
except Exception:
|
|
419
|
+
return # Silently ignore errors in log streaming
|
|
420
|
+
|
|
421
|
+
# Stream logs continuously
|
|
422
|
+
if exec_result:
|
|
423
|
+
try:
|
|
424
|
+
for line in exec_result.output:
|
|
425
|
+
if stop_requested.is_set():
|
|
426
|
+
break
|
|
427
|
+
|
|
428
|
+
decoded_line = line.decode("utf-8").rstrip()
|
|
429
|
+
|
|
430
|
+
# Skip tail error messages about files not existing yet
|
|
431
|
+
if "tail:" in decoded_line and (
|
|
432
|
+
"cannot open" in decoded_line or "no files remaining" in decoded_line
|
|
433
|
+
):
|
|
434
|
+
continue
|
|
435
|
+
|
|
436
|
+
# Replace file path headers with friendly names
|
|
437
|
+
for file_path, friendly_name in log_names.items():
|
|
438
|
+
log_header = f"==> {file_path} <=="
|
|
439
|
+
if log_header in decoded_line:
|
|
440
|
+
decoded_line = decoded_line.replace(log_header, f"==> {friendly_name} <==")
|
|
441
|
+
|
|
442
|
+
# Print "ready" message when container becomes healthy
|
|
443
|
+
if container_ready.is_set() and not hasattr(stream_logs_with_health_check, "ready_printed"):
|
|
444
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
445
|
+
click.echo(
|
|
446
|
+
FeedbackManager.highlight(message="» Watching Tinybird Local... (Press Ctrl+C to stop)")
|
|
447
|
+
)
|
|
448
|
+
stream_logs_with_health_check.ready_printed = True # type: ignore
|
|
449
|
+
|
|
450
|
+
click.echo(decoded_line)
|
|
451
|
+
except Exception:
|
|
452
|
+
pass # Silently ignore errors when stream is interrupted
|
|
453
|
+
|
|
454
|
+
log_thread = threading.Thread(target=stream_logs_with_health_check, daemon=True)
|
|
455
|
+
log_thread.start()
|
|
456
|
+
|
|
457
|
+
health_check_thread = threading.Thread(target=check_endpoints_health, daemon=True)
|
|
458
|
+
health_check_thread.start()
|
|
459
|
+
|
|
460
|
+
# Monitor container health in main thread
|
|
461
|
+
try:
|
|
462
|
+
while True:
|
|
463
|
+
container.reload() # Refresh container attributes
|
|
464
|
+
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
465
|
+
if health == "healthy":
|
|
466
|
+
container_ready.set()
|
|
467
|
+
# Keep monitoring and streaming logs until Ctrl+C or health check failure
|
|
468
|
+
while True:
|
|
469
|
+
# Check if health check detected an error
|
|
470
|
+
if stop_requested.is_set() and health_check_error["message"]:
|
|
471
|
+
time.sleep(0.5) # Give log thread time to finish printing
|
|
472
|
+
click.echo(
|
|
473
|
+
FeedbackManager.error(
|
|
474
|
+
message=f"\n✗ {health_check_error['message']}\n"
|
|
475
|
+
"Please run `tb local restart` to restart the container."
|
|
476
|
+
)
|
|
477
|
+
)
|
|
478
|
+
return
|
|
479
|
+
time.sleep(1)
|
|
480
|
+
if health == "unhealthy":
|
|
481
|
+
stop_requested.set()
|
|
482
|
+
raise CLILocalException(
|
|
483
|
+
FeedbackManager.error(
|
|
484
|
+
message="Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
|
|
485
|
+
)
|
|
486
|
+
)
|
|
487
|
+
time.sleep(5)
|
|
488
|
+
except KeyboardInterrupt:
|
|
489
|
+
stop_requested.set()
|
|
490
|
+
click.echo(FeedbackManager.highlight(message="» Stopping Tinybird Local..."))
|
|
491
|
+
container.stop()
|
|
492
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local stopped."))
|
|
493
|
+
return
|
|
494
|
+
|
|
495
|
+
# Non-watch mode: just wait for container to be healthy
|
|
302
496
|
while True:
|
|
303
497
|
container.reload() # Refresh container attributes
|
|
304
498
|
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
@@ -310,7 +504,6 @@ def start_tinybird_local(
|
|
|
310
504
|
message="Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
|
|
311
505
|
)
|
|
312
506
|
)
|
|
313
|
-
|
|
314
507
|
time.sleep(5)
|
|
315
508
|
|
|
316
509
|
# Remove tinybird-local dangling images to avoid running out of disk space
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
tinybird/connectors.py,sha256=TSux0opkG3gbzTTyRIUfNk4RJoLvTIyp42rdsmmIAYg,15083
|
|
2
|
-
tinybird/context.py,sha256=
|
|
2
|
+
tinybird/context.py,sha256=QNS9yB3i3bQUZMc_Z4oBN_oMDZaBM-Q6g12jNLjNH98,1261
|
|
3
3
|
tinybird/datatypes.py,sha256=r4WCvspmrXTJHiPjjyOTiZyZl31FO3Ynkwq4LQsYm6E,11059
|
|
4
4
|
tinybird/feedback_manager.py,sha256=XY8d83pRlq-LH7xHMApkaEebfXEWLjDzrGe1prpcTHE,69778
|
|
5
5
|
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
6
|
-
tinybird/prompts.py,sha256=
|
|
7
|
-
tinybird/service_datasources.py,sha256=
|
|
6
|
+
tinybird/prompts.py,sha256=F8ic_mGId50MDJmg5ec8i5BDavxz9SWtocLXqgO0IRY,45689
|
|
7
|
+
tinybird/service_datasources.py,sha256=jFjRYbGykd76UcFNX7rry4bdPNDxl4SIbMkMVClyk54,54711
|
|
8
8
|
tinybird/sql.py,sha256=7pkwQMUVy0CH5zFgLMZyCx1BeaJSQYC05EmOb9vO3Ls,48694
|
|
9
9
|
tinybird/sql_template.py,sha256=qz1Q-d8jlGpU8w-jcwn_NM4ApFtJAhEL-3_3-JELrl4,102979
|
|
10
10
|
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
@@ -18,35 +18,36 @@ tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1w
|
|
|
18
18
|
tinybird/datafile/parse_connection.py,sha256=tRyn2Rpr1TeWet5BXmMoQgaotbGdYep1qiTak_OqC5E,1825
|
|
19
19
|
tinybird/datafile/parse_datasource.py,sha256=ssW8QeFSgglVFi3sDZj_HgkJiTJ2069v2JgqnH3CkDE,1825
|
|
20
20
|
tinybird/datafile/parse_pipe.py,sha256=8e9LMecSQVWHC4AXf8cdxoQ1nxUR4fTObYxTctO_EXQ,3816
|
|
21
|
-
tinybird/tb/__cli__.py,sha256=
|
|
21
|
+
tinybird/tb/__cli__.py,sha256=sGbfRkjI4JkVGzo0JQPjoiJ4aJczl7ueJiW4NshluI4,247
|
|
22
22
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
23
|
-
tinybird/tb/cli.py,sha256=
|
|
24
|
-
tinybird/tb/client.py,sha256=
|
|
23
|
+
tinybird/tb/cli.py,sha256=Os4uApzYYVPXcYFdaReQs9E12q9DOuYOUVFoOgt7i7U,1047
|
|
24
|
+
tinybird/tb/client.py,sha256=VYbllzIsmBS4b56Ix-vsyLbhpbe7MVRxR1sIo6xU2qA,53544
|
|
25
25
|
tinybird/tb/config.py,sha256=XIKju3xxpo40oQ48zDtSv0vgUrBZBHSmHJi4SHksjXE,5447
|
|
26
|
-
tinybird/tb/modules/build.py,sha256=
|
|
26
|
+
tinybird/tb/modules/build.py,sha256=gnZoqxEzFDc8OKVzmKIaesy3tat0lbJ3E8JKmsKTutw,8321
|
|
27
27
|
tinybird/tb/modules/build_common.py,sha256=zHqnzEdJvEIlzDGEL5HqcUXInuKLqYG6hPCdB6f6PMA,20064
|
|
28
28
|
tinybird/tb/modules/cicd.py,sha256=0KLKccha9IP749QvlXBmzdWv1On3mFwMY4DUcJlBxiE,7326
|
|
29
|
-
tinybird/tb/modules/cli.py,sha256=
|
|
30
|
-
tinybird/tb/modules/common.py,sha256=
|
|
29
|
+
tinybird/tb/modules/cli.py,sha256=YsnFZT3WbeFX-zhlB9t10czHKzRJBoutzLZZe1OqUCw,21240
|
|
30
|
+
tinybird/tb/modules/common.py,sha256=vSfyJCdmoUvO1B_JmODqKeMYQy4VXOYrFhBue758VMw,88275
|
|
31
31
|
tinybird/tb/modules/config.py,sha256=gK7rgaWTDd4ZKCrNEg_Uemr26EQjqWt6TjyQKujxOws,11462
|
|
32
32
|
tinybird/tb/modules/connection.py,sha256=axp8Fny1_4PSLJGN4UF6WygyRbQtM3Lbt6thxHKTxzw,17790
|
|
33
33
|
tinybird/tb/modules/copy.py,sha256=dPZkcIDvxjJrlQUIvToO0vsEEEs4EYumbNV77-BzNoU,4404
|
|
34
34
|
tinybird/tb/modules/create.py,sha256=j0WvO6LxuGW1ZOYVuvR1FxFls2oMmKePYNTRJMSXfuU,20135
|
|
35
35
|
tinybird/tb/modules/datasource.py,sha256=O-w4qwhAIitD_SLVdxkpknckliRp_xMswdB8n8Ohxo0,42335
|
|
36
36
|
tinybird/tb/modules/deployment.py,sha256=v0layOmG0IMnuXc3RT39mpGfa5M8yPlrL9F089fJFCo,15964
|
|
37
|
-
tinybird/tb/modules/deployment_common.py,sha256=
|
|
37
|
+
tinybird/tb/modules/deployment_common.py,sha256=68qoCoOQFqRFyvxWwYmKiuVb37z7txBhEvzSOJGUHMc,20664
|
|
38
38
|
tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
|
|
39
39
|
tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
|
|
40
40
|
tinybird/tb/modules/endpoint.py,sha256=ksRj6mfDb9Xv63PhTkV_uKSosgysHElqagg3RTt21Do,11958
|
|
41
|
+
tinybird/tb/modules/environment.py,sha256=z99321d_imk3eLJx9JNZIWMVcaRPUNixUV8pom4OL5o,5724
|
|
41
42
|
tinybird/tb/modules/exceptions.py,sha256=_1BHy0OixEkeF0fOCu1_1Pj8pJS4BNfUyucqCViJGTw,5958
|
|
42
|
-
tinybird/tb/modules/feedback_manager.py,sha256=
|
|
43
|
+
tinybird/tb/modules/feedback_manager.py,sha256=mSlsC1dN0Val0tDcUP8EXcnFVNXnyjBdmDhKVSc2qR0,78145
|
|
43
44
|
tinybird/tb/modules/info.py,sha256=UKlazKWSE3bzEtggTwyDhmkh7WRrWURuBraR36ClbmQ,7177
|
|
44
45
|
tinybird/tb/modules/infra.py,sha256=J9Noe9aZo5Y9ZKAhqh9jnv8azfivXLLHQ2a9TeMWN9s,32673
|
|
45
46
|
tinybird/tb/modules/job.py,sha256=wBsnu8UPTOha2rkLvucgmw4xYv73ubmui3eeSIF68ZM,3107
|
|
46
47
|
tinybird/tb/modules/llm.py,sha256=fPBBCmM3KlCksLlgJkg4joDn6y3H5QjDzE-Pm4YNf7E,1782
|
|
47
48
|
tinybird/tb/modules/llm_utils.py,sha256=nS9r4FAElJw8yXtmdYrx-rtI2zXR8qXfi1QqUDCfxvg,3469
|
|
48
|
-
tinybird/tb/modules/local.py,sha256=
|
|
49
|
-
tinybird/tb/modules/local_common.py,sha256=
|
|
49
|
+
tinybird/tb/modules/local.py,sha256=Fw5yTJBzGc_d4nYKCzjxEfDZogc4P-2nQJf5_rNRwuA,8616
|
|
50
|
+
tinybird/tb/modules/local_common.py,sha256=3WLYEm8bUik6Z2YFiWmzfhOpuChb-Iyy0cnYQoCnRIA,28302
|
|
50
51
|
tinybird/tb/modules/login.py,sha256=q2fc0dy5brwofQSNQppipS1aDcyrxHvVjZZbzIA3b4g,1916
|
|
51
52
|
tinybird/tb/modules/login_common.py,sha256=u1Eh1u6L7V3x8qmxVO-LnWDELHWAnXm3bvb7BNjVXF8,11990
|
|
52
53
|
tinybird/tb/modules/logout.py,sha256=sniI4JNxpTrVeRCp0oGJuQ3yRerG4hH5uz6oBmjv724,1009
|
|
@@ -121,8 +122,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
121
122
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
122
123
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
123
124
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
124
|
-
tinybird-0.0.1.
|
|
125
|
-
tinybird-0.0.1.
|
|
126
|
-
tinybird-0.0.1.
|
|
127
|
-
tinybird-0.0.1.
|
|
128
|
-
tinybird-0.0.1.
|
|
125
|
+
tinybird-0.0.1.dev307.dist-info/METADATA,sha256=F1RA-wrl0jqMAo128B0tMdVBrCTW5zEC1EulkwKSyHo,1881
|
|
126
|
+
tinybird-0.0.1.dev307.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
127
|
+
tinybird-0.0.1.dev307.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
128
|
+
tinybird-0.0.1.dev307.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
129
|
+
tinybird-0.0.1.dev307.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|