tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/ch_utils/constants.py +5 -0
- tinybird/connectors.py +1 -7
- tinybird/context.py +3 -3
- tinybird/datafile/common.py +10 -8
- tinybird/datafile/parse_pipe.py +2 -2
- tinybird/feedback_manager.py +3 -0
- tinybird/prompts.py +1 -0
- tinybird/service_datasources.py +223 -0
- tinybird/sql_template.py +26 -11
- tinybird/sql_template_fmt.py +14 -4
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/cli.py +1 -0
- tinybird/tb/client.py +104 -26
- tinybird/tb/config.py +24 -0
- tinybird/tb/modules/agent/agent.py +103 -67
- tinybird/tb/modules/agent/banner.py +15 -15
- tinybird/tb/modules/agent/explore_agent.py +5 -0
- tinybird/tb/modules/agent/mock_agent.py +5 -1
- tinybird/tb/modules/agent/models.py +6 -2
- tinybird/tb/modules/agent/prompts.py +49 -2
- tinybird/tb/modules/agent/tools/deploy.py +1 -1
- tinybird/tb/modules/agent/tools/execute_query.py +15 -18
- tinybird/tb/modules/agent/tools/request_endpoint.py +1 -1
- tinybird/tb/modules/agent/tools/run_command.py +9 -0
- tinybird/tb/modules/agent/utils.py +38 -48
- tinybird/tb/modules/branch.py +150 -0
- tinybird/tb/modules/build.py +58 -13
- tinybird/tb/modules/build_common.py +209 -25
- tinybird/tb/modules/cli.py +129 -16
- tinybird/tb/modules/common.py +172 -146
- tinybird/tb/modules/connection.py +125 -194
- tinybird/tb/modules/connection_kafka.py +382 -0
- tinybird/tb/modules/copy.py +3 -1
- tinybird/tb/modules/create.py +83 -150
- tinybird/tb/modules/datafile/build.py +27 -38
- tinybird/tb/modules/datafile/build_datasource.py +21 -25
- tinybird/tb/modules/datafile/diff.py +1 -1
- tinybird/tb/modules/datafile/format_pipe.py +46 -7
- tinybird/tb/modules/datafile/playground.py +59 -68
- tinybird/tb/modules/datafile/pull.py +2 -3
- tinybird/tb/modules/datasource.py +477 -308
- tinybird/tb/modules/deployment.py +2 -0
- tinybird/tb/modules/deployment_common.py +84 -44
- tinybird/tb/modules/deprecations.py +4 -4
- tinybird/tb/modules/dev_server.py +33 -12
- tinybird/tb/modules/exceptions.py +14 -0
- tinybird/tb/modules/feedback_manager.py +1 -1
- tinybird/tb/modules/info.py +69 -12
- tinybird/tb/modules/infra.py +4 -5
- tinybird/tb/modules/job_common.py +15 -0
- tinybird/tb/modules/local.py +143 -23
- tinybird/tb/modules/local_common.py +347 -19
- tinybird/tb/modules/local_logs.py +209 -0
- tinybird/tb/modules/login.py +21 -2
- tinybird/tb/modules/login_common.py +254 -12
- tinybird/tb/modules/mock.py +5 -54
- tinybird/tb/modules/mock_common.py +0 -54
- tinybird/tb/modules/open.py +10 -5
- tinybird/tb/modules/project.py +14 -5
- tinybird/tb/modules/shell.py +15 -7
- tinybird/tb/modules/sink.py +3 -1
- tinybird/tb/modules/telemetry.py +11 -3
- tinybird/tb/modules/test.py +13 -9
- tinybird/tb/modules/test_common.py +13 -87
- tinybird/tb/modules/tinyunit/tinyunit.py +0 -14
- tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -6
- tinybird/tb/modules/watch.py +5 -3
- tinybird/tb_cli_modules/common.py +2 -2
- tinybird/tb_cli_modules/telemetry.py +1 -1
- tinybird/tornado_template.py +6 -7
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/METADATA +32 -6
- tinybird-1.0.5.dist-info/RECORD +132 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/WHEEL +1 -1
- tinybird-0.0.1.dev291.dist-info/RECORD +0 -128
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/local.py
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import uuid
|
|
1
3
|
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
2
5
|
|
|
3
6
|
import click
|
|
7
|
+
import jwt
|
|
4
8
|
import requests
|
|
5
|
-
|
|
6
9
|
from docker.client import DockerClient
|
|
10
|
+
|
|
7
11
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.common import update_cli
|
|
12
|
+
from tinybird.tb.modules.common import echo_json, update_cli
|
|
13
|
+
from tinybird.tb.modules.exceptions import CLILocalException
|
|
9
14
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
15
|
+
from tinybird.tb.modules.info import get_local_info
|
|
10
16
|
from tinybird.tb.modules.local_common import (
|
|
11
17
|
TB_CONTAINER_NAME,
|
|
12
18
|
TB_LOCAL_ADDRESS,
|
|
@@ -14,6 +20,19 @@ from tinybird.tb.modules.local_common import (
|
|
|
14
20
|
get_existing_container_with_matching_env,
|
|
15
21
|
start_tinybird_local,
|
|
16
22
|
)
|
|
23
|
+
from tinybird.tb.modules.local_logs import (
|
|
24
|
+
check_memory_sufficient,
|
|
25
|
+
clickhouse_is_ready,
|
|
26
|
+
container_is_ready,
|
|
27
|
+
container_is_starting,
|
|
28
|
+
container_is_stopping,
|
|
29
|
+
container_is_unhealthy,
|
|
30
|
+
container_stats,
|
|
31
|
+
events_is_ready,
|
|
32
|
+
local_authentication_is_ready,
|
|
33
|
+
redis_is_ready,
|
|
34
|
+
server_is_ready,
|
|
35
|
+
)
|
|
17
36
|
|
|
18
37
|
|
|
19
38
|
def stop_tinybird_local(docker_client: DockerClient) -> None:
|
|
@@ -72,30 +91,69 @@ def stop() -> None:
|
|
|
72
91
|
@click.pass_context
|
|
73
92
|
def status(ctx: click.Context) -> None:
|
|
74
93
|
"""Check status of Tinybird Local"""
|
|
94
|
+
|
|
95
|
+
click.echo(FeedbackManager.highlight(message="» Checking status..."))
|
|
75
96
|
docker_client = get_docker_client()
|
|
76
97
|
container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, {})
|
|
77
98
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
99
|
+
try:
|
|
100
|
+
if container:
|
|
101
|
+
if container_is_ready(container):
|
|
102
|
+
stats = container_stats(container, docker_client)
|
|
103
|
+
click.echo(FeedbackManager.info(message=f"✓ Tinybird Local container ({stats})"))
|
|
104
|
+
|
|
105
|
+
# Check memory sufficiency
|
|
106
|
+
is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
|
|
107
|
+
if not is_sufficient and warning_msg:
|
|
108
|
+
click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
|
|
109
|
+
|
|
110
|
+
if not clickhouse_is_ready(container):
|
|
111
|
+
raise Exception("Clickhouse is not ready.")
|
|
112
|
+
click.echo(FeedbackManager.info(message="✓ Clickhouse"))
|
|
113
|
+
|
|
114
|
+
if not redis_is_ready(container):
|
|
115
|
+
raise Exception("Redis is not ready.")
|
|
116
|
+
click.echo(FeedbackManager.info(message="✓ Redis"))
|
|
117
|
+
|
|
118
|
+
if not server_is_ready(container):
|
|
119
|
+
raise Exception("Server is not ready.")
|
|
120
|
+
click.echo(FeedbackManager.info(message="✓ Server"))
|
|
121
|
+
|
|
122
|
+
if not events_is_ready(container):
|
|
123
|
+
raise Exception("Events is not ready.")
|
|
124
|
+
click.echo(FeedbackManager.info(message="✓ Events"))
|
|
125
|
+
|
|
126
|
+
if not local_authentication_is_ready(container):
|
|
127
|
+
raise Exception("Tinybird Local authentication is not ready.")
|
|
128
|
+
click.echo(FeedbackManager.info(message="✓ Tinybird Local authentication"))
|
|
129
|
+
|
|
130
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
131
|
+
click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
|
|
132
|
+
config = ctx.ensure_object(dict).get("config", {})
|
|
133
|
+
get_local_info(config)
|
|
134
|
+
elif container_is_starting(container):
|
|
135
|
+
click.echo(FeedbackManager.highlight(message="* Tinybird Local is starting..."))
|
|
136
|
+
elif container_is_stopping(container):
|
|
137
|
+
click.echo(FeedbackManager.highlight(message="* Tinybird Local is stopping..."))
|
|
138
|
+
elif container_is_unhealthy(container):
|
|
139
|
+
is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
|
|
140
|
+
if not is_sufficient and warning_msg:
|
|
141
|
+
click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
|
|
142
|
+
click.echo(
|
|
143
|
+
FeedbackManager.error(
|
|
144
|
+
message="* Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
|
|
145
|
+
)
|
|
146
|
+
)
|
|
147
|
+
else:
|
|
148
|
+
click.echo(
|
|
149
|
+
FeedbackManager.error(message="✗ Tinybird Local is not running. Run 'tb local start' to start it")
|
|
150
|
+
)
|
|
93
151
|
else:
|
|
94
152
|
click.echo(
|
|
95
|
-
FeedbackManager.
|
|
153
|
+
FeedbackManager.error(message="✗ Tinybird Local is not running. Run 'tb local start' to start it")
|
|
96
154
|
)
|
|
97
|
-
|
|
98
|
-
|
|
155
|
+
except Exception as e:
|
|
156
|
+
raise CLILocalException(FeedbackManager.error(message=f"Tinybird Local is not ready. Reason: {e}"))
|
|
99
157
|
|
|
100
158
|
|
|
101
159
|
@local.command()
|
|
@@ -125,7 +183,33 @@ def remove() -> None:
|
|
|
125
183
|
is_flag=True,
|
|
126
184
|
help="Skip pulling the latest Tinybird Local image. Use directly your current local image.",
|
|
127
185
|
)
|
|
128
|
-
|
|
186
|
+
@click.option(
|
|
187
|
+
"--user-token",
|
|
188
|
+
default=None,
|
|
189
|
+
envvar="TB_LOCAL_USER_TOKEN",
|
|
190
|
+
help="User token to use for the Tinybird Local container.",
|
|
191
|
+
)
|
|
192
|
+
@click.option(
|
|
193
|
+
"--workspace-token",
|
|
194
|
+
default=None,
|
|
195
|
+
envvar="TB_LOCAL_WORKSPACE_TOKEN",
|
|
196
|
+
help="Workspace token to use for the Tinybird Local container.",
|
|
197
|
+
)
|
|
198
|
+
@click.option(
|
|
199
|
+
"--daemon/--watch",
|
|
200
|
+
"-d/-w",
|
|
201
|
+
default=False,
|
|
202
|
+
is_flag=True,
|
|
203
|
+
help="Run Tinybird Local in the background.",
|
|
204
|
+
)
|
|
205
|
+
def start(
|
|
206
|
+
use_aws_creds: bool,
|
|
207
|
+
volumes_path: str,
|
|
208
|
+
skip_new_version: bool,
|
|
209
|
+
user_token: str,
|
|
210
|
+
workspace_token: str,
|
|
211
|
+
daemon: bool,
|
|
212
|
+
) -> None:
|
|
129
213
|
"""Start Tinybird Local"""
|
|
130
214
|
if volumes_path is not None:
|
|
131
215
|
absolute_path = Path(volumes_path).absolute()
|
|
@@ -134,8 +218,12 @@ def start(use_aws_creds: bool, volumes_path: str, skip_new_version: bool) -> Non
|
|
|
134
218
|
|
|
135
219
|
click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
|
|
136
220
|
docker_client = get_docker_client()
|
|
137
|
-
|
|
138
|
-
|
|
221
|
+
watch = not daemon
|
|
222
|
+
start_tinybird_local(
|
|
223
|
+
docker_client, use_aws_creds, volumes_path, skip_new_version, user_token, workspace_token, watch=watch
|
|
224
|
+
)
|
|
225
|
+
if daemon:
|
|
226
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
139
227
|
|
|
140
228
|
|
|
141
229
|
@local.command()
|
|
@@ -183,3 +271,35 @@ def version() -> None:
|
|
|
183
271
|
"""Show Tinybird Local version"""
|
|
184
272
|
response = requests.get(f"{TB_LOCAL_ADDRESS}/version")
|
|
185
273
|
click.echo(FeedbackManager.success(message=f"✓ Tinybird Local version: {response.text}"))
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
@local.command()
|
|
277
|
+
@click.pass_context
|
|
278
|
+
def generate_tokens(ctx: click.Context) -> None:
|
|
279
|
+
"""Generate static tokens for initializing Tinybird Local"""
|
|
280
|
+
output = ctx.ensure_object(dict).get("output")
|
|
281
|
+
user_id = str(uuid.uuid4())
|
|
282
|
+
workspace_id = str(uuid.uuid4())
|
|
283
|
+
user_token_id = str(uuid.uuid4())
|
|
284
|
+
workspace_token_id = str(uuid.uuid4())
|
|
285
|
+
payload = {"u": user_id, "id": user_token_id, "host": None}
|
|
286
|
+
user_token = generate_token(payload)
|
|
287
|
+
payload = {"u": workspace_id, "id": workspace_token_id, "host": None}
|
|
288
|
+
workspace_token = generate_token(payload)
|
|
289
|
+
|
|
290
|
+
if output == "json":
|
|
291
|
+
echo_json({"workspace_token": workspace_token, "user_token": user_token})
|
|
292
|
+
else:
|
|
293
|
+
click.echo(FeedbackManager.gray(message="Workspace token: ") + FeedbackManager.info(message=workspace_token))
|
|
294
|
+
click.echo(FeedbackManager.gray(message="User token: ") + FeedbackManager.info(message=user_token))
|
|
295
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local tokens generated!"))
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def generate_token(payload: dict[str, Any]) -> str:
|
|
299
|
+
algo = jwt.algorithms.get_default_algorithms()["HS256"]
|
|
300
|
+
msg = json.dumps(payload)
|
|
301
|
+
msg_base64 = jwt.utils.base64url_encode(msg.encode())
|
|
302
|
+
sign_key = algo.prepare_key("abcd")
|
|
303
|
+
signature = algo.sign(msg_base64, sign_key)
|
|
304
|
+
token = msg_base64 + b"." + jwt.utils.base64url_encode(signature)
|
|
305
|
+
return "p." + token.decode()
|
|
@@ -4,20 +4,31 @@ import logging
|
|
|
4
4
|
import os
|
|
5
5
|
import re
|
|
6
6
|
import subprocess
|
|
7
|
+
import threading
|
|
7
8
|
import time
|
|
9
|
+
import uuid
|
|
8
10
|
from typing import Any, Dict, Optional
|
|
9
11
|
|
|
10
12
|
import boto3
|
|
11
13
|
import click
|
|
12
14
|
import requests
|
|
13
|
-
|
|
14
|
-
import docker
|
|
15
15
|
from docker.client import DockerClient
|
|
16
16
|
from docker.models.containers import Container
|
|
17
|
+
|
|
18
|
+
import docker
|
|
17
19
|
from tinybird.tb.client import AuthNoTokenException, TinyB
|
|
18
20
|
from tinybird.tb.modules.config import CLIConfig
|
|
19
21
|
from tinybird.tb.modules.exceptions import CLILocalException
|
|
20
22
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
23
|
+
from tinybird.tb.modules.local_logs import (
|
|
24
|
+
check_memory_sufficient,
|
|
25
|
+
clickhouse_is_ready,
|
|
26
|
+
container_stats,
|
|
27
|
+
events_is_ready,
|
|
28
|
+
local_authentication_is_ready,
|
|
29
|
+
redis_is_ready,
|
|
30
|
+
server_is_ready,
|
|
31
|
+
)
|
|
21
32
|
from tinybird.tb.modules.secret_common import load_secrets
|
|
22
33
|
from tinybird.tb.modules.telemetry import add_telemetry_event
|
|
23
34
|
|
|
@@ -34,11 +45,18 @@ def get_tinybird_local_client(
|
|
|
34
45
|
config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
|
|
35
46
|
) -> TinyB:
|
|
36
47
|
"""Get a Tinybird client connected to the local environment."""
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
48
|
+
try:
|
|
49
|
+
config = get_tinybird_local_config(config_obj, test=test, silent=silent)
|
|
50
|
+
client = config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
|
|
51
|
+
load_secrets(config_obj.get("path", ""), client)
|
|
52
|
+
return client
|
|
53
|
+
# if some of the API calls to tinybird local fail due to a JSONDecodeError, it means that container is running but it's unhealthy
|
|
54
|
+
except json.JSONDecodeError:
|
|
55
|
+
raise CLILocalException(
|
|
56
|
+
message=FeedbackManager.error(
|
|
57
|
+
message="Tinybird Local is running but it's unhealthy. Please check if it's running and try again. If the problem persists, please run `tb local restart` and try again."
|
|
58
|
+
)
|
|
59
|
+
)
|
|
42
60
|
|
|
43
61
|
|
|
44
62
|
def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
|
|
@@ -56,6 +74,18 @@ def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, si
|
|
|
56
74
|
if path:
|
|
57
75
|
user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
|
|
58
76
|
if test:
|
|
77
|
+
# delete any Tinybird_Local_Test_* workspace
|
|
78
|
+
user_workspaces = requests.get(
|
|
79
|
+
f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?with_organization=true&token={admin_token}"
|
|
80
|
+
).json()
|
|
81
|
+
local_workspaces = user_workspaces.get("workspaces", [])
|
|
82
|
+
for ws in local_workspaces:
|
|
83
|
+
is_test_workspace = ws["name"].startswith("Tinybird_Local_Test_")
|
|
84
|
+
if is_test_workspace:
|
|
85
|
+
requests.delete(
|
|
86
|
+
f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
|
|
87
|
+
)
|
|
88
|
+
|
|
59
89
|
ws_name = get_test_workspace_name(path)
|
|
60
90
|
else:
|
|
61
91
|
ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
|
|
@@ -104,8 +134,8 @@ def get_build_workspace_name(path: str) -> str:
|
|
|
104
134
|
|
|
105
135
|
|
|
106
136
|
def get_test_workspace_name(path: str) -> str:
|
|
107
|
-
|
|
108
|
-
return f"Tinybird_Local_Test_{
|
|
137
|
+
random_folder_suffix = str(uuid.uuid4()).replace("-", "_")
|
|
138
|
+
return f"Tinybird_Local_Test_{random_folder_suffix}"
|
|
109
139
|
|
|
110
140
|
|
|
111
141
|
def get_local_tokens() -> Dict[str, str]:
|
|
@@ -127,6 +157,7 @@ def get_local_tokens() -> Dict[str, str]:
|
|
|
127
157
|
},
|
|
128
158
|
)
|
|
129
159
|
|
|
160
|
+
# TODO: If docker errors persist, explain that you can use custom environments too once they are open for everyone
|
|
130
161
|
if container and container.status == "running":
|
|
131
162
|
if container.health == "healthy":
|
|
132
163
|
raise CLILocalException(
|
|
@@ -206,7 +237,7 @@ def get_local_tokens() -> Dict[str, str]:
|
|
|
206
237
|
default=True,
|
|
207
238
|
)
|
|
208
239
|
if yes:
|
|
209
|
-
click.echo(FeedbackManager.highlight(message="»
|
|
240
|
+
click.echo(FeedbackManager.highlight(message="» Watching Tinybird Local... (Press Ctrl+C to stop)"))
|
|
210
241
|
docker_client = get_docker_client()
|
|
211
242
|
start_tinybird_local(docker_client, False)
|
|
212
243
|
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
@@ -221,7 +252,10 @@ def start_tinybird_local(
|
|
|
221
252
|
docker_client: DockerClient,
|
|
222
253
|
use_aws_creds: bool,
|
|
223
254
|
volumes_path: Optional[str] = None,
|
|
224
|
-
skip_new_version: bool =
|
|
255
|
+
skip_new_version: bool = True,
|
|
256
|
+
user_token: Optional[str] = None,
|
|
257
|
+
workspace_token: Optional[str] = None,
|
|
258
|
+
watch: bool = False,
|
|
225
259
|
) -> None:
|
|
226
260
|
"""Start the Tinybird container."""
|
|
227
261
|
pull_show_prompt = False
|
|
@@ -248,7 +282,13 @@ def start_tinybird_local(
|
|
|
248
282
|
if pull_required:
|
|
249
283
|
docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
|
|
250
284
|
|
|
251
|
-
environment =
|
|
285
|
+
environment = {}
|
|
286
|
+
if use_aws_creds:
|
|
287
|
+
environment.update(get_use_aws_creds())
|
|
288
|
+
if user_token:
|
|
289
|
+
environment["TB_LOCAL_USER_TOKEN"] = user_token
|
|
290
|
+
if workspace_token:
|
|
291
|
+
environment["TB_LOCAL_WORKSPACE_TOKEN"] = workspace_token
|
|
252
292
|
|
|
253
293
|
container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, environment)
|
|
254
294
|
|
|
@@ -278,18 +318,151 @@ def start_tinybird_local(
|
|
|
278
318
|
)
|
|
279
319
|
|
|
280
320
|
click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
|
|
321
|
+
|
|
322
|
+
if watch:
|
|
323
|
+
# Stream logs in a separate thread while monitoring container health
|
|
324
|
+
container_ready = threading.Event()
|
|
325
|
+
stop_requested = threading.Event()
|
|
326
|
+
health_check: dict[str, str] = {}
|
|
327
|
+
|
|
328
|
+
log_thread = threading.Thread(
|
|
329
|
+
target=stream_logs_with_health_check,
|
|
330
|
+
args=(container, container_ready, stop_requested),
|
|
331
|
+
daemon=True,
|
|
332
|
+
)
|
|
333
|
+
log_thread.start()
|
|
334
|
+
|
|
335
|
+
health_check_thread = threading.Thread(
|
|
336
|
+
target=check_endpoints_health,
|
|
337
|
+
args=(container, docker_client, container_ready, stop_requested, health_check),
|
|
338
|
+
daemon=True,
|
|
339
|
+
)
|
|
340
|
+
health_check_thread.start()
|
|
341
|
+
|
|
342
|
+
# Monitor container health in main thread
|
|
343
|
+
memory_warning_shown = False
|
|
344
|
+
try:
|
|
345
|
+
while True:
|
|
346
|
+
container.reload() # Refresh container attributes
|
|
347
|
+
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
348
|
+
if not container_ready.is_set():
|
|
349
|
+
click.echo(FeedbackManager.info(message=f"* Tinybird Local container status: {health}"))
|
|
350
|
+
stats = container_stats(container, docker_client)
|
|
351
|
+
click.echo(f"* {stats}")
|
|
352
|
+
|
|
353
|
+
# Check memory sufficiency
|
|
354
|
+
if not memory_warning_shown:
|
|
355
|
+
is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
|
|
356
|
+
if not is_sufficient and warning_msg:
|
|
357
|
+
click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
|
|
358
|
+
memory_warning_shown = True
|
|
359
|
+
|
|
360
|
+
if health == "healthy":
|
|
361
|
+
click.echo(FeedbackManager.highlight(message="» Checking services..."))
|
|
362
|
+
stats = container_stats(container, docker_client)
|
|
363
|
+
click.echo(FeedbackManager.info(message=f"✓ Tinybird Local container ({stats})"))
|
|
364
|
+
|
|
365
|
+
# Check memory sufficiency before checking services
|
|
366
|
+
if not memory_warning_shown:
|
|
367
|
+
is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
|
|
368
|
+
if not is_sufficient and warning_msg:
|
|
369
|
+
click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
|
|
370
|
+
memory_warning_shown = True
|
|
371
|
+
|
|
372
|
+
if not clickhouse_is_ready(container):
|
|
373
|
+
raise Exception("Clickhouse is not ready.")
|
|
374
|
+
click.echo(FeedbackManager.info(message="✓ Clickhouse"))
|
|
375
|
+
|
|
376
|
+
if not redis_is_ready(container):
|
|
377
|
+
raise Exception("Redis is not ready.")
|
|
378
|
+
click.echo(FeedbackManager.info(message="✓ Redis"))
|
|
379
|
+
|
|
380
|
+
if not server_is_ready(container):
|
|
381
|
+
raise Exception("Server is not ready.")
|
|
382
|
+
click.echo(FeedbackManager.info(message="✓ Server"))
|
|
383
|
+
|
|
384
|
+
if not events_is_ready(container):
|
|
385
|
+
raise Exception("Events is not ready.")
|
|
386
|
+
click.echo(FeedbackManager.info(message="✓ Events"))
|
|
387
|
+
|
|
388
|
+
if not local_authentication_is_ready(container):
|
|
389
|
+
raise Exception("Tinybird Local authentication is not ready.")
|
|
390
|
+
click.echo(FeedbackManager.info(message="✓ Tinybird Local authentication"))
|
|
391
|
+
container_ready.set()
|
|
392
|
+
# Keep monitoring and streaming logs until Ctrl+C or health check failure
|
|
393
|
+
while True:
|
|
394
|
+
# Check if health check detected an error
|
|
395
|
+
if stop_requested.is_set() and health_check.get("error"):
|
|
396
|
+
time.sleep(0.5) # Give log thread time to finish printing
|
|
397
|
+
raise CLILocalException(
|
|
398
|
+
FeedbackManager.error(
|
|
399
|
+
message=f"{health_check.get('error')}\n"
|
|
400
|
+
"Please run `tb local restart` to restart the container."
|
|
401
|
+
)
|
|
402
|
+
)
|
|
403
|
+
return
|
|
404
|
+
time.sleep(1)
|
|
405
|
+
if health == "unhealthy":
|
|
406
|
+
stop_requested.set()
|
|
407
|
+
# Check if memory might be the cause of unhealthy status
|
|
408
|
+
is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
|
|
409
|
+
error_msg = "Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
|
|
410
|
+
if not is_sufficient and warning_msg:
|
|
411
|
+
error_msg = (
|
|
412
|
+
"Tinybird Local is unhealthy.\nnAfter adjusting memory, try running `tb local restart`."
|
|
413
|
+
)
|
|
414
|
+
raise CLILocalException(FeedbackManager.error(message=error_msg))
|
|
415
|
+
time.sleep(5)
|
|
416
|
+
except KeyboardInterrupt:
|
|
417
|
+
stop_requested.set()
|
|
418
|
+
click.echo(FeedbackManager.highlight(message="» Stopping Tinybird Local..."))
|
|
419
|
+
try:
|
|
420
|
+
container.stop()
|
|
421
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local stopped."))
|
|
422
|
+
except KeyboardInterrupt:
|
|
423
|
+
click.echo(FeedbackManager.warning(message="⚠ Forced exit. Container may still be running."))
|
|
424
|
+
click.echo(FeedbackManager.info(message=" Run `tb local stop` to stop the container manually."))
|
|
425
|
+
return
|
|
426
|
+
|
|
427
|
+
# Non-watch mode: just wait for container to be healthy
|
|
428
|
+
memory_warning_shown = False
|
|
281
429
|
while True:
|
|
282
430
|
container.reload() # Refresh container attributes
|
|
283
431
|
health = container.attrs.get("State", {}).get("Health", {}).get("Status")
|
|
432
|
+
click.echo(FeedbackManager.info(message=f"* Tinybird Local container status: {health}"))
|
|
433
|
+
stats = container_stats(container, docker_client)
|
|
434
|
+
click.echo(f"* {stats}")
|
|
435
|
+
|
|
436
|
+
# Check memory sufficiency
|
|
437
|
+
if not memory_warning_shown:
|
|
438
|
+
is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
|
|
439
|
+
if not is_sufficient and warning_msg:
|
|
440
|
+
click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
|
|
441
|
+
memory_warning_shown = True
|
|
442
|
+
|
|
284
443
|
if health == "healthy":
|
|
444
|
+
click.echo(FeedbackManager.highlight(message="» Checking services..."))
|
|
445
|
+
stats = container_stats(container, docker_client)
|
|
446
|
+
click.echo(FeedbackManager.info(message=f"✓ Tinybird Local container ({stats})"))
|
|
447
|
+
if not clickhouse_is_ready(container):
|
|
448
|
+
raise Exception("Clickhouse is not ready.")
|
|
449
|
+
click.echo(FeedbackManager.info(message="✓ Clickhouse"))
|
|
450
|
+
if not redis_is_ready(container):
|
|
451
|
+
raise Exception("Redis is not ready.")
|
|
452
|
+
click.echo(FeedbackManager.info(message="✓ Redis"))
|
|
453
|
+
if not server_is_ready(container):
|
|
454
|
+
raise Exception("Server is not ready.")
|
|
455
|
+
click.echo(FeedbackManager.info(message="✓ Server"))
|
|
456
|
+
if not events_is_ready(container):
|
|
457
|
+
raise Exception("Events is not ready.")
|
|
458
|
+
click.echo(FeedbackManager.info(message="✓ Events"))
|
|
459
|
+
if not local_authentication_is_ready(container):
|
|
460
|
+
raise Exception("Tinybird Local authentication is not ready.")
|
|
461
|
+
click.echo(FeedbackManager.info(message="✓ Tinybird Local authentication"))
|
|
285
462
|
break
|
|
286
463
|
if health == "unhealthy":
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
message="Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
|
|
290
|
-
)
|
|
291
|
-
)
|
|
292
|
-
|
|
464
|
+
error_msg = "Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
|
|
465
|
+
raise CLILocalException(FeedbackManager.error(message=error_msg))
|
|
293
466
|
time.sleep(5)
|
|
294
467
|
|
|
295
468
|
# Remove tinybird-local dangling images to avoid running out of disk space
|
|
@@ -408,7 +581,8 @@ def get_docker_client() -> DockerClient:
|
|
|
408
581
|
message=(
|
|
409
582
|
f"No container runtime is running. Make sure a Docker-compatible runtime is installed and running. "
|
|
410
583
|
f"{docker_location_message}\n\n"
|
|
411
|
-
"If you're using a custom location, please provide it using the DOCKER_HOST environment variable
|
|
584
|
+
"If you're using a custom location, please provide it using the DOCKER_HOST environment variable.\n\n"
|
|
585
|
+
"Alternatively, you can use Tinybird branches to develop your project without Docker. Run `tb branch create my_feature_branch` to create one. Learn more at: https://www.tinybird.co/docs/forward/test-and-deploy/branches"
|
|
412
586
|
)
|
|
413
587
|
)
|
|
414
588
|
)
|
|
@@ -453,3 +627,157 @@ def get_use_aws_creds() -> dict[str, str]:
|
|
|
453
627
|
)
|
|
454
628
|
|
|
455
629
|
return credentials
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
SERVICE_COLORS = {
|
|
633
|
+
"[EVENTS]": "\033[95m", # Magenta
|
|
634
|
+
"[SERVER]": "\033[94m", # Blue
|
|
635
|
+
"[HEALTH]": "\033[96m", # Cyan
|
|
636
|
+
"[KAFKA]": "\033[93m", # Yellow
|
|
637
|
+
"[AUTH]": "\033[90m", # Gray
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
RESET = "\033[0m"
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def check_endpoints_health(
|
|
644
|
+
container: Container,
|
|
645
|
+
docker_client: DockerClient,
|
|
646
|
+
container_ready: threading.Event,
|
|
647
|
+
stop_requested: threading.Event,
|
|
648
|
+
health_check: dict[str, str],
|
|
649
|
+
) -> None:
|
|
650
|
+
"""Continuously check /tokens and /v0/health endpoints"""
|
|
651
|
+
# Wait for container to be ready before starting health checks
|
|
652
|
+
container_ready.wait()
|
|
653
|
+
|
|
654
|
+
# Give container a moment to fully start up
|
|
655
|
+
time.sleep(2)
|
|
656
|
+
|
|
657
|
+
check_interval = 10 # Check every 10 seconds
|
|
658
|
+
|
|
659
|
+
while not stop_requested.is_set():
|
|
660
|
+
try:
|
|
661
|
+
# Check /tokens endpoint
|
|
662
|
+
tokens_response = requests.get(f"{TB_LOCAL_ADDRESS}/tokens", timeout=5)
|
|
663
|
+
if tokens_response.status_code != 200:
|
|
664
|
+
health_check["error"] = (
|
|
665
|
+
f"/tokens endpoint returned status {tokens_response.status_code}. Tinybird Local may be unhealthy."
|
|
666
|
+
)
|
|
667
|
+
stop_requested.set()
|
|
668
|
+
break
|
|
669
|
+
|
|
670
|
+
# Check /v0/health endpoint
|
|
671
|
+
health_response = requests.get(f"{TB_LOCAL_ADDRESS}/v0/health", timeout=5)
|
|
672
|
+
if health_response.status_code != 200:
|
|
673
|
+
health_check["error"] = (
|
|
674
|
+
f"/v0/health endpoint returned status {health_response.status_code}. "
|
|
675
|
+
"Tinybird Local may be unhealthy."
|
|
676
|
+
)
|
|
677
|
+
stop_requested.set()
|
|
678
|
+
break
|
|
679
|
+
|
|
680
|
+
# Verify tokens response has expected structure
|
|
681
|
+
try:
|
|
682
|
+
tokens_data = tokens_response.json()
|
|
683
|
+
if not all(key in tokens_data for key in ["user_token", "admin_token", "workspace_admin_token"]):
|
|
684
|
+
health_check["error"] = (
|
|
685
|
+
"/tokens endpoint returned unexpected data. Tinybird Local may be unhealthy."
|
|
686
|
+
)
|
|
687
|
+
stop_requested.set()
|
|
688
|
+
break
|
|
689
|
+
except json.JSONDecodeError:
|
|
690
|
+
health_check["error"] = "/tokens endpoint returned invalid JSON. Tinybird Local may be unhealthy."
|
|
691
|
+
stop_requested.set()
|
|
692
|
+
break
|
|
693
|
+
|
|
694
|
+
except Exception as e:
|
|
695
|
+
# Check if it's a connection error
|
|
696
|
+
error_str = str(e)
|
|
697
|
+
if "connect" in error_str.lower() or "timeout" in error_str.lower():
|
|
698
|
+
health_check["error"] = f"Failed to connect to Tinybird Local: {error_str}"
|
|
699
|
+
else:
|
|
700
|
+
health_check["error"] = f"Health check failed: {error_str}"
|
|
701
|
+
stop_requested.set()
|
|
702
|
+
break
|
|
703
|
+
|
|
704
|
+
if container_ready.is_set():
|
|
705
|
+
stats = container_stats(container, docker_client)
|
|
706
|
+
click.echo(f"{SERVICE_COLORS['[HEALTH]']}[HEALTH]{RESET} {stats}")
|
|
707
|
+
|
|
708
|
+
# Wait before next check
|
|
709
|
+
for _ in range(check_interval):
|
|
710
|
+
if stop_requested.is_set():
|
|
711
|
+
break
|
|
712
|
+
time.sleep(1)
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def stream_logs_with_health_check(
|
|
716
|
+
container: Container, container_ready: threading.Event, stop_requested: threading.Event
|
|
717
|
+
) -> None:
|
|
718
|
+
"""Stream logs and monitor container health in parallel"""
|
|
719
|
+
# Wait for container to be ready before starting health checks
|
|
720
|
+
container_ready.wait()
|
|
721
|
+
|
|
722
|
+
# Give container a moment to fully start up
|
|
723
|
+
time.sleep(2)
|
|
724
|
+
|
|
725
|
+
retry_count = 0
|
|
726
|
+
max_retries = 10
|
|
727
|
+
exec_result = None
|
|
728
|
+
|
|
729
|
+
while retry_count < max_retries and not stop_requested.is_set():
|
|
730
|
+
try:
|
|
731
|
+
# Try to tail the log files (only new logs, not historical)
|
|
732
|
+
# Use -F to follow by name and retry if files don't exist yet
|
|
733
|
+
log_files = {
|
|
734
|
+
"/var/log/tinybird-local-server.log": "SERVER",
|
|
735
|
+
"/var/log/tinybird-local-hfi.log": "EVENTS",
|
|
736
|
+
"/var/log/tinybird-local-setup.log": "AUTH",
|
|
737
|
+
"/var/log/tinybird-local-kafka.log": "KAFKA",
|
|
738
|
+
}
|
|
739
|
+
# Build commands to tail each file and prefix with its label (using stdbuf for unbuffered output)
|
|
740
|
+
tail_commands = [
|
|
741
|
+
f'tail -n 0 -f {path} | stdbuf -oL sed "s/^/[{source}] /"' for path, source in log_files.items()
|
|
742
|
+
]
|
|
743
|
+
# Join with & to run in parallel, then wait for all
|
|
744
|
+
cmd = f"sh -c '({' & '.join(tail_commands)}) & wait'"
|
|
745
|
+
exec_result = container.exec_run(cmd, stream=True, tty=False, stdout=True, stderr=True)
|
|
746
|
+
break # Success, exit retry loop
|
|
747
|
+
except Exception:
|
|
748
|
+
# Log file might not exist yet, wait and retry
|
|
749
|
+
retry_count += 1
|
|
750
|
+
if retry_count < max_retries:
|
|
751
|
+
time.sleep(2)
|
|
752
|
+
|
|
753
|
+
# Stream logs continuously
|
|
754
|
+
if exec_result:
|
|
755
|
+
try:
|
|
756
|
+
for line in exec_result.output:
|
|
757
|
+
if stop_requested.is_set():
|
|
758
|
+
break
|
|
759
|
+
|
|
760
|
+
raw_line = line.decode("utf-8").rstrip()
|
|
761
|
+
lines = raw_line.split("\n")
|
|
762
|
+
|
|
763
|
+
# Print "ready" message when container becomes healthy
|
|
764
|
+
if container_ready.is_set() and not hasattr(stream_logs_with_health_check, "ready_printed"):
|
|
765
|
+
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
766
|
+
stream_logs_with_health_check.ready_printed = True # type: ignore
|
|
767
|
+
|
|
768
|
+
for line in lines:
|
|
769
|
+
# Apply color to service label
|
|
770
|
+
for service, color in SERVICE_COLORS.items():
|
|
771
|
+
if line.startswith(service):
|
|
772
|
+
message = line[len(service) :]
|
|
773
|
+
# extract content of message="...""
|
|
774
|
+
match = re.search(r'message="([^"]*)"', message)
|
|
775
|
+
if match:
|
|
776
|
+
message = match.group(1)
|
|
777
|
+
line = f"{color}{service}{RESET} {message}"
|
|
778
|
+
break
|
|
779
|
+
|
|
780
|
+
click.echo(line)
|
|
781
|
+
|
|
782
|
+
except Exception:
|
|
783
|
+
pass # Silently ignore errors when stream is interrupted
|