tinybird 0.0.1.dev291__py3-none-any.whl → 1.0.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. tinybird/ch_utils/constants.py +5 -0
  2. tinybird/connectors.py +1 -7
  3. tinybird/context.py +3 -3
  4. tinybird/datafile/common.py +10 -8
  5. tinybird/datafile/parse_pipe.py +2 -2
  6. tinybird/feedback_manager.py +3 -0
  7. tinybird/prompts.py +1 -0
  8. tinybird/service_datasources.py +223 -0
  9. tinybird/sql_template.py +26 -11
  10. tinybird/sql_template_fmt.py +14 -4
  11. tinybird/tb/__cli__.py +2 -2
  12. tinybird/tb/cli.py +1 -0
  13. tinybird/tb/client.py +104 -26
  14. tinybird/tb/config.py +24 -0
  15. tinybird/tb/modules/agent/agent.py +103 -67
  16. tinybird/tb/modules/agent/banner.py +15 -15
  17. tinybird/tb/modules/agent/explore_agent.py +5 -0
  18. tinybird/tb/modules/agent/mock_agent.py +5 -1
  19. tinybird/tb/modules/agent/models.py +6 -2
  20. tinybird/tb/modules/agent/prompts.py +49 -2
  21. tinybird/tb/modules/agent/tools/deploy.py +1 -1
  22. tinybird/tb/modules/agent/tools/execute_query.py +15 -18
  23. tinybird/tb/modules/agent/tools/request_endpoint.py +1 -1
  24. tinybird/tb/modules/agent/tools/run_command.py +9 -0
  25. tinybird/tb/modules/agent/utils.py +38 -48
  26. tinybird/tb/modules/branch.py +150 -0
  27. tinybird/tb/modules/build.py +58 -13
  28. tinybird/tb/modules/build_common.py +209 -25
  29. tinybird/tb/modules/cli.py +129 -16
  30. tinybird/tb/modules/common.py +172 -146
  31. tinybird/tb/modules/connection.py +125 -194
  32. tinybird/tb/modules/connection_kafka.py +382 -0
  33. tinybird/tb/modules/copy.py +3 -1
  34. tinybird/tb/modules/create.py +83 -150
  35. tinybird/tb/modules/datafile/build.py +27 -38
  36. tinybird/tb/modules/datafile/build_datasource.py +21 -25
  37. tinybird/tb/modules/datafile/diff.py +1 -1
  38. tinybird/tb/modules/datafile/format_pipe.py +46 -7
  39. tinybird/tb/modules/datafile/playground.py +59 -68
  40. tinybird/tb/modules/datafile/pull.py +2 -3
  41. tinybird/tb/modules/datasource.py +477 -308
  42. tinybird/tb/modules/deployment.py +2 -0
  43. tinybird/tb/modules/deployment_common.py +84 -44
  44. tinybird/tb/modules/deprecations.py +4 -4
  45. tinybird/tb/modules/dev_server.py +33 -12
  46. tinybird/tb/modules/exceptions.py +14 -0
  47. tinybird/tb/modules/feedback_manager.py +1 -1
  48. tinybird/tb/modules/info.py +69 -12
  49. tinybird/tb/modules/infra.py +4 -5
  50. tinybird/tb/modules/job_common.py +15 -0
  51. tinybird/tb/modules/local.py +143 -23
  52. tinybird/tb/modules/local_common.py +347 -19
  53. tinybird/tb/modules/local_logs.py +209 -0
  54. tinybird/tb/modules/login.py +21 -2
  55. tinybird/tb/modules/login_common.py +254 -12
  56. tinybird/tb/modules/mock.py +5 -54
  57. tinybird/tb/modules/mock_common.py +0 -54
  58. tinybird/tb/modules/open.py +10 -5
  59. tinybird/tb/modules/project.py +14 -5
  60. tinybird/tb/modules/shell.py +15 -7
  61. tinybird/tb/modules/sink.py +3 -1
  62. tinybird/tb/modules/telemetry.py +11 -3
  63. tinybird/tb/modules/test.py +13 -9
  64. tinybird/tb/modules/test_common.py +13 -87
  65. tinybird/tb/modules/tinyunit/tinyunit.py +0 -14
  66. tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -6
  67. tinybird/tb/modules/watch.py +5 -3
  68. tinybird/tb_cli_modules/common.py +2 -2
  69. tinybird/tb_cli_modules/telemetry.py +1 -1
  70. tinybird/tornado_template.py +6 -7
  71. {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/METADATA +32 -6
  72. tinybird-1.0.5.dist-info/RECORD +132 -0
  73. {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/WHEEL +1 -1
  74. tinybird-0.0.1.dev291.dist-info/RECORD +0 -128
  75. {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/entry_points.txt +0 -0
  76. {tinybird-0.0.1.dev291.dist-info → tinybird-1.0.5.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,18 @@
1
+ import json
2
+ import uuid
1
3
  from pathlib import Path
4
+ from typing import Any
2
5
 
3
6
  import click
7
+ import jwt
4
8
  import requests
5
-
6
9
  from docker.client import DockerClient
10
+
7
11
  from tinybird.tb.modules.cli import cli
8
- from tinybird.tb.modules.common import update_cli
12
+ from tinybird.tb.modules.common import echo_json, update_cli
13
+ from tinybird.tb.modules.exceptions import CLILocalException
9
14
  from tinybird.tb.modules.feedback_manager import FeedbackManager
15
+ from tinybird.tb.modules.info import get_local_info
10
16
  from tinybird.tb.modules.local_common import (
11
17
  TB_CONTAINER_NAME,
12
18
  TB_LOCAL_ADDRESS,
@@ -14,6 +20,19 @@ from tinybird.tb.modules.local_common import (
14
20
  get_existing_container_with_matching_env,
15
21
  start_tinybird_local,
16
22
  )
23
+ from tinybird.tb.modules.local_logs import (
24
+ check_memory_sufficient,
25
+ clickhouse_is_ready,
26
+ container_is_ready,
27
+ container_is_starting,
28
+ container_is_stopping,
29
+ container_is_unhealthy,
30
+ container_stats,
31
+ events_is_ready,
32
+ local_authentication_is_ready,
33
+ redis_is_ready,
34
+ server_is_ready,
35
+ )
17
36
 
18
37
 
19
38
  def stop_tinybird_local(docker_client: DockerClient) -> None:
@@ -72,30 +91,69 @@ def stop() -> None:
72
91
  @click.pass_context
73
92
  def status(ctx: click.Context) -> None:
74
93
  """Check status of Tinybird Local"""
94
+
95
+ click.echo(FeedbackManager.highlight(message="» Checking status..."))
75
96
  docker_client = get_docker_client()
76
97
  container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, {})
77
98
 
78
- if container:
79
- status = container.status
80
- health = container.attrs.get("State", {}).get("Health", {}).get("Status")
81
-
82
- if status == "running" and health == "healthy":
83
- click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
84
- click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
85
- from tinybird.tb.modules.info import get_local_info
86
-
87
- config = ctx.ensure_object(dict).get("config", {})
88
- get_local_info(config)
89
- elif status == "restarting" or (status == "running" and health == "starting"):
90
- click.echo(FeedbackManager.highlight(message="* Tinybird Local is starting..."))
91
- elif status == "removing":
92
- click.echo(FeedbackManager.highlight(message="* Tinybird Local is stopping..."))
99
+ try:
100
+ if container:
101
+ if container_is_ready(container):
102
+ stats = container_stats(container, docker_client)
103
+ click.echo(FeedbackManager.info(message=f" Tinybird Local container ({stats})"))
104
+
105
+ # Check memory sufficiency
106
+ is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
107
+ if not is_sufficient and warning_msg:
108
+ click.echo(FeedbackManager.warning(message=f" {warning_msg}"))
109
+
110
+ if not clickhouse_is_ready(container):
111
+ raise Exception("Clickhouse is not ready.")
112
+ click.echo(FeedbackManager.info(message="✓ Clickhouse"))
113
+
114
+ if not redis_is_ready(container):
115
+ raise Exception("Redis is not ready.")
116
+ click.echo(FeedbackManager.info(message="✓ Redis"))
117
+
118
+ if not server_is_ready(container):
119
+ raise Exception("Server is not ready.")
120
+ click.echo(FeedbackManager.info(message="✓ Server"))
121
+
122
+ if not events_is_ready(container):
123
+ raise Exception("Events is not ready.")
124
+ click.echo(FeedbackManager.info(message="✓ Events"))
125
+
126
+ if not local_authentication_is_ready(container):
127
+ raise Exception("Tinybird Local authentication is not ready.")
128
+ click.echo(FeedbackManager.info(message="✓ Tinybird Local authentication"))
129
+
130
+ click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
131
+ click.echo(FeedbackManager.highlight(message="\n» Tinybird Local:"))
132
+ config = ctx.ensure_object(dict).get("config", {})
133
+ get_local_info(config)
134
+ elif container_is_starting(container):
135
+ click.echo(FeedbackManager.highlight(message="* Tinybird Local is starting..."))
136
+ elif container_is_stopping(container):
137
+ click.echo(FeedbackManager.highlight(message="* Tinybird Local is stopping..."))
138
+ elif container_is_unhealthy(container):
139
+ is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
140
+ if not is_sufficient and warning_msg:
141
+ click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
142
+ click.echo(
143
+ FeedbackManager.error(
144
+ message="* Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
145
+ )
146
+ )
147
+ else:
148
+ click.echo(
149
+ FeedbackManager.error(message="✗ Tinybird Local is not running. Run 'tb local start' to start it")
150
+ )
93
151
  else:
94
152
  click.echo(
95
- FeedbackManager.info(message="✗ Tinybird Local is not running. Run 'tb local start' to start it")
153
+ FeedbackManager.error(message="✗ Tinybird Local is not running. Run 'tb local start' to start it")
96
154
  )
97
- else:
98
- click.echo(FeedbackManager.info(message="Tinybird Local is not running. Run 'tb local start' to start it"))
155
+ except Exception as e:
156
+ raise CLILocalException(FeedbackManager.error(message=f"Tinybird Local is not ready. Reason: {e}"))
99
157
 
100
158
 
101
159
  @local.command()
@@ -125,7 +183,33 @@ def remove() -> None:
125
183
  is_flag=True,
126
184
  help="Skip pulling the latest Tinybird Local image. Use directly your current local image.",
127
185
  )
128
- def start(use_aws_creds: bool, volumes_path: str, skip_new_version: bool) -> None:
186
+ @click.option(
187
+ "--user-token",
188
+ default=None,
189
+ envvar="TB_LOCAL_USER_TOKEN",
190
+ help="User token to use for the Tinybird Local container.",
191
+ )
192
+ @click.option(
193
+ "--workspace-token",
194
+ default=None,
195
+ envvar="TB_LOCAL_WORKSPACE_TOKEN",
196
+ help="Workspace token to use for the Tinybird Local container.",
197
+ )
198
+ @click.option(
199
+ "--daemon/--watch",
200
+ "-d/-w",
201
+ default=False,
202
+ is_flag=True,
203
+ help="Run Tinybird Local in the background.",
204
+ )
205
+ def start(
206
+ use_aws_creds: bool,
207
+ volumes_path: str,
208
+ skip_new_version: bool,
209
+ user_token: str,
210
+ workspace_token: str,
211
+ daemon: bool,
212
+ ) -> None:
129
213
  """Start Tinybird Local"""
130
214
  if volumes_path is not None:
131
215
  absolute_path = Path(volumes_path).absolute()
@@ -134,8 +218,12 @@ def start(use_aws_creds: bool, volumes_path: str, skip_new_version: bool) -> Non
134
218
 
135
219
  click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
136
220
  docker_client = get_docker_client()
137
- start_tinybird_local(docker_client, use_aws_creds, volumes_path, skip_new_version)
138
- click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
221
+ watch = not daemon
222
+ start_tinybird_local(
223
+ docker_client, use_aws_creds, volumes_path, skip_new_version, user_token, workspace_token, watch=watch
224
+ )
225
+ if daemon:
226
+ click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
139
227
 
140
228
 
141
229
  @local.command()
@@ -183,3 +271,35 @@ def version() -> None:
183
271
  """Show Tinybird Local version"""
184
272
  response = requests.get(f"{TB_LOCAL_ADDRESS}/version")
185
273
  click.echo(FeedbackManager.success(message=f"✓ Tinybird Local version: {response.text}"))
274
+
275
+
276
+ @local.command()
277
+ @click.pass_context
278
+ def generate_tokens(ctx: click.Context) -> None:
279
+ """Generate static tokens for initializing Tinybird Local"""
280
+ output = ctx.ensure_object(dict).get("output")
281
+ user_id = str(uuid.uuid4())
282
+ workspace_id = str(uuid.uuid4())
283
+ user_token_id = str(uuid.uuid4())
284
+ workspace_token_id = str(uuid.uuid4())
285
+ payload = {"u": user_id, "id": user_token_id, "host": None}
286
+ user_token = generate_token(payload)
287
+ payload = {"u": workspace_id, "id": workspace_token_id, "host": None}
288
+ workspace_token = generate_token(payload)
289
+
290
+ if output == "json":
291
+ echo_json({"workspace_token": workspace_token, "user_token": user_token})
292
+ else:
293
+ click.echo(FeedbackManager.gray(message="Workspace token: ") + FeedbackManager.info(message=workspace_token))
294
+ click.echo(FeedbackManager.gray(message="User token: ") + FeedbackManager.info(message=user_token))
295
+ click.echo(FeedbackManager.success(message="✓ Tinybird Local tokens generated!"))
296
+
297
+
298
+ def generate_token(payload: dict[str, Any]) -> str:
299
+ algo = jwt.algorithms.get_default_algorithms()["HS256"]
300
+ msg = json.dumps(payload)
301
+ msg_base64 = jwt.utils.base64url_encode(msg.encode())
302
+ sign_key = algo.prepare_key("abcd")
303
+ signature = algo.sign(msg_base64, sign_key)
304
+ token = msg_base64 + b"." + jwt.utils.base64url_encode(signature)
305
+ return "p." + token.decode()
@@ -4,20 +4,31 @@ import logging
4
4
  import os
5
5
  import re
6
6
  import subprocess
7
+ import threading
7
8
  import time
9
+ import uuid
8
10
  from typing import Any, Dict, Optional
9
11
 
10
12
  import boto3
11
13
  import click
12
14
  import requests
13
-
14
- import docker
15
15
  from docker.client import DockerClient
16
16
  from docker.models.containers import Container
17
+
18
+ import docker
17
19
  from tinybird.tb.client import AuthNoTokenException, TinyB
18
20
  from tinybird.tb.modules.config import CLIConfig
19
21
  from tinybird.tb.modules.exceptions import CLILocalException
20
22
  from tinybird.tb.modules.feedback_manager import FeedbackManager
23
+ from tinybird.tb.modules.local_logs import (
24
+ check_memory_sufficient,
25
+ clickhouse_is_ready,
26
+ container_stats,
27
+ events_is_ready,
28
+ local_authentication_is_ready,
29
+ redis_is_ready,
30
+ server_is_ready,
31
+ )
21
32
  from tinybird.tb.modules.secret_common import load_secrets
22
33
  from tinybird.tb.modules.telemetry import add_telemetry_event
23
34
 
@@ -34,11 +45,18 @@ def get_tinybird_local_client(
34
45
  config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
35
46
  ) -> TinyB:
36
47
  """Get a Tinybird client connected to the local environment."""
37
-
38
- config = get_tinybird_local_config(config_obj, test=test, silent=silent)
39
- client = config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
40
- load_secrets(config_obj.get("path", ""), client)
41
- return client
48
+ try:
49
+ config = get_tinybird_local_config(config_obj, test=test, silent=silent)
50
+ client = config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
51
+ load_secrets(config_obj.get("path", ""), client)
52
+ return client
53
+ # if some of the API calls to tinybird local fail due to a JSONDecodeError, it means that container is running but it's unhealthy
54
+ except json.JSONDecodeError:
55
+ raise CLILocalException(
56
+ message=FeedbackManager.error(
57
+ message="Tinybird Local is running but it's unhealthy. Please check if it's running and try again. If the problem persists, please run `tb local restart` and try again."
58
+ )
59
+ )
42
60
 
43
61
 
44
62
  def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
@@ -56,6 +74,18 @@ def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, si
56
74
  if path:
57
75
  user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
58
76
  if test:
77
+ # delete any Tinybird_Local_Test_* workspace
78
+ user_workspaces = requests.get(
79
+ f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?with_organization=true&token={admin_token}"
80
+ ).json()
81
+ local_workspaces = user_workspaces.get("workspaces", [])
82
+ for ws in local_workspaces:
83
+ is_test_workspace = ws["name"].startswith("Tinybird_Local_Test_")
84
+ if is_test_workspace:
85
+ requests.delete(
86
+ f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
87
+ )
88
+
59
89
  ws_name = get_test_workspace_name(path)
60
90
  else:
61
91
  ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
@@ -104,8 +134,8 @@ def get_build_workspace_name(path: str) -> str:
104
134
 
105
135
 
106
136
  def get_test_workspace_name(path: str) -> str:
107
- folder_hash = hashlib.sha256(path.encode()).hexdigest()
108
- return f"Tinybird_Local_Test_{folder_hash}"
137
+ random_folder_suffix = str(uuid.uuid4()).replace("-", "_")
138
+ return f"Tinybird_Local_Test_{random_folder_suffix}"
109
139
 
110
140
 
111
141
  def get_local_tokens() -> Dict[str, str]:
@@ -127,6 +157,7 @@ def get_local_tokens() -> Dict[str, str]:
127
157
  },
128
158
  )
129
159
 
160
+ # TODO: If docker errors persist, explain that you can use custom environments too once they are open for everyone
130
161
  if container and container.status == "running":
131
162
  if container.health == "healthy":
132
163
  raise CLILocalException(
@@ -206,7 +237,7 @@ def get_local_tokens() -> Dict[str, str]:
206
237
  default=True,
207
238
  )
208
239
  if yes:
209
- click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
240
+ click.echo(FeedbackManager.highlight(message="» Watching Tinybird Local... (Press Ctrl+C to stop)"))
210
241
  docker_client = get_docker_client()
211
242
  start_tinybird_local(docker_client, False)
212
243
  click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
@@ -221,7 +252,10 @@ def start_tinybird_local(
221
252
  docker_client: DockerClient,
222
253
  use_aws_creds: bool,
223
254
  volumes_path: Optional[str] = None,
224
- skip_new_version: bool = False,
255
+ skip_new_version: bool = True,
256
+ user_token: Optional[str] = None,
257
+ workspace_token: Optional[str] = None,
258
+ watch: bool = False,
225
259
  ) -> None:
226
260
  """Start the Tinybird container."""
227
261
  pull_show_prompt = False
@@ -248,7 +282,13 @@ def start_tinybird_local(
248
282
  if pull_required:
249
283
  docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
250
284
 
251
- environment = get_use_aws_creds() if use_aws_creds else {}
285
+ environment = {}
286
+ if use_aws_creds:
287
+ environment.update(get_use_aws_creds())
288
+ if user_token:
289
+ environment["TB_LOCAL_USER_TOKEN"] = user_token
290
+ if workspace_token:
291
+ environment["TB_LOCAL_WORKSPACE_TOKEN"] = workspace_token
252
292
 
253
293
  container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, environment)
254
294
 
@@ -278,18 +318,151 @@ def start_tinybird_local(
278
318
  )
279
319
 
280
320
  click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
321
+
322
+ if watch:
323
+ # Stream logs in a separate thread while monitoring container health
324
+ container_ready = threading.Event()
325
+ stop_requested = threading.Event()
326
+ health_check: dict[str, str] = {}
327
+
328
+ log_thread = threading.Thread(
329
+ target=stream_logs_with_health_check,
330
+ args=(container, container_ready, stop_requested),
331
+ daemon=True,
332
+ )
333
+ log_thread.start()
334
+
335
+ health_check_thread = threading.Thread(
336
+ target=check_endpoints_health,
337
+ args=(container, docker_client, container_ready, stop_requested, health_check),
338
+ daemon=True,
339
+ )
340
+ health_check_thread.start()
341
+
342
+ # Monitor container health in main thread
343
+ memory_warning_shown = False
344
+ try:
345
+ while True:
346
+ container.reload() # Refresh container attributes
347
+ health = container.attrs.get("State", {}).get("Health", {}).get("Status")
348
+ if not container_ready.is_set():
349
+ click.echo(FeedbackManager.info(message=f"* Tinybird Local container status: {health}"))
350
+ stats = container_stats(container, docker_client)
351
+ click.echo(f"* {stats}")
352
+
353
+ # Check memory sufficiency
354
+ if not memory_warning_shown:
355
+ is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
356
+ if not is_sufficient and warning_msg:
357
+ click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
358
+ memory_warning_shown = True
359
+
360
+ if health == "healthy":
361
+ click.echo(FeedbackManager.highlight(message="» Checking services..."))
362
+ stats = container_stats(container, docker_client)
363
+ click.echo(FeedbackManager.info(message=f"✓ Tinybird Local container ({stats})"))
364
+
365
+ # Check memory sufficiency before checking services
366
+ if not memory_warning_shown:
367
+ is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
368
+ if not is_sufficient and warning_msg:
369
+ click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
370
+ memory_warning_shown = True
371
+
372
+ if not clickhouse_is_ready(container):
373
+ raise Exception("Clickhouse is not ready.")
374
+ click.echo(FeedbackManager.info(message="✓ Clickhouse"))
375
+
376
+ if not redis_is_ready(container):
377
+ raise Exception("Redis is not ready.")
378
+ click.echo(FeedbackManager.info(message="✓ Redis"))
379
+
380
+ if not server_is_ready(container):
381
+ raise Exception("Server is not ready.")
382
+ click.echo(FeedbackManager.info(message="✓ Server"))
383
+
384
+ if not events_is_ready(container):
385
+ raise Exception("Events is not ready.")
386
+ click.echo(FeedbackManager.info(message="✓ Events"))
387
+
388
+ if not local_authentication_is_ready(container):
389
+ raise Exception("Tinybird Local authentication is not ready.")
390
+ click.echo(FeedbackManager.info(message="✓ Tinybird Local authentication"))
391
+ container_ready.set()
392
+ # Keep monitoring and streaming logs until Ctrl+C or health check failure
393
+ while True:
394
+ # Check if health check detected an error
395
+ if stop_requested.is_set() and health_check.get("error"):
396
+ time.sleep(0.5) # Give log thread time to finish printing
397
+ raise CLILocalException(
398
+ FeedbackManager.error(
399
+ message=f"{health_check.get('error')}\n"
400
+ "Please run `tb local restart` to restart the container."
401
+ )
402
+ )
403
+ return
404
+ time.sleep(1)
405
+ if health == "unhealthy":
406
+ stop_requested.set()
407
+ # Check if memory might be the cause of unhealthy status
408
+ is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
409
+ error_msg = "Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
410
+ if not is_sufficient and warning_msg:
411
+ error_msg = (
412
+ "Tinybird Local is unhealthy.\nnAfter adjusting memory, try running `tb local restart`."
413
+ )
414
+ raise CLILocalException(FeedbackManager.error(message=error_msg))
415
+ time.sleep(5)
416
+ except KeyboardInterrupt:
417
+ stop_requested.set()
418
+ click.echo(FeedbackManager.highlight(message="» Stopping Tinybird Local..."))
419
+ try:
420
+ container.stop()
421
+ click.echo(FeedbackManager.success(message="✓ Tinybird Local stopped."))
422
+ except KeyboardInterrupt:
423
+ click.echo(FeedbackManager.warning(message="⚠ Forced exit. Container may still be running."))
424
+ click.echo(FeedbackManager.info(message=" Run `tb local stop` to stop the container manually."))
425
+ return
426
+
427
+ # Non-watch mode: just wait for container to be healthy
428
+ memory_warning_shown = False
281
429
  while True:
282
430
  container.reload() # Refresh container attributes
283
431
  health = container.attrs.get("State", {}).get("Health", {}).get("Status")
432
+ click.echo(FeedbackManager.info(message=f"* Tinybird Local container status: {health}"))
433
+ stats = container_stats(container, docker_client)
434
+ click.echo(f"* {stats}")
435
+
436
+ # Check memory sufficiency
437
+ if not memory_warning_shown:
438
+ is_sufficient, warning_msg = check_memory_sufficient(container, docker_client)
439
+ if not is_sufficient and warning_msg:
440
+ click.echo(FeedbackManager.warning(message=f"△ {warning_msg}"))
441
+ memory_warning_shown = True
442
+
284
443
  if health == "healthy":
444
+ click.echo(FeedbackManager.highlight(message="» Checking services..."))
445
+ stats = container_stats(container, docker_client)
446
+ click.echo(FeedbackManager.info(message=f"✓ Tinybird Local container ({stats})"))
447
+ if not clickhouse_is_ready(container):
448
+ raise Exception("Clickhouse is not ready.")
449
+ click.echo(FeedbackManager.info(message="✓ Clickhouse"))
450
+ if not redis_is_ready(container):
451
+ raise Exception("Redis is not ready.")
452
+ click.echo(FeedbackManager.info(message="✓ Redis"))
453
+ if not server_is_ready(container):
454
+ raise Exception("Server is not ready.")
455
+ click.echo(FeedbackManager.info(message="✓ Server"))
456
+ if not events_is_ready(container):
457
+ raise Exception("Events is not ready.")
458
+ click.echo(FeedbackManager.info(message="✓ Events"))
459
+ if not local_authentication_is_ready(container):
460
+ raise Exception("Tinybird Local authentication is not ready.")
461
+ click.echo(FeedbackManager.info(message="✓ Tinybird Local authentication"))
285
462
  break
286
463
  if health == "unhealthy":
287
- raise CLILocalException(
288
- FeedbackManager.error(
289
- message="Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
290
- )
291
- )
292
-
464
+ error_msg = "Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
465
+ raise CLILocalException(FeedbackManager.error(message=error_msg))
293
466
  time.sleep(5)
294
467
 
295
468
  # Remove tinybird-local dangling images to avoid running out of disk space
@@ -408,7 +581,8 @@ def get_docker_client() -> DockerClient:
408
581
  message=(
409
582
  f"No container runtime is running. Make sure a Docker-compatible runtime is installed and running. "
410
583
  f"{docker_location_message}\n\n"
411
- "If you're using a custom location, please provide it using the DOCKER_HOST environment variable."
584
+ "If you're using a custom location, please provide it using the DOCKER_HOST environment variable.\n\n"
585
+ "Alternatively, you can use Tinybird branches to develop your project without Docker. Run `tb branch create my_feature_branch` to create one. Learn more at: https://www.tinybird.co/docs/forward/test-and-deploy/branches"
412
586
  )
413
587
  )
414
588
  )
@@ -453,3 +627,157 @@ def get_use_aws_creds() -> dict[str, str]:
453
627
  )
454
628
 
455
629
  return credentials
630
+
631
+
632
+ SERVICE_COLORS = {
633
+ "[EVENTS]": "\033[95m", # Magenta
634
+ "[SERVER]": "\033[94m", # Blue
635
+ "[HEALTH]": "\033[96m", # Cyan
636
+ "[KAFKA]": "\033[93m", # Yellow
637
+ "[AUTH]": "\033[90m", # Gray
638
+ }
639
+
640
+ RESET = "\033[0m"
641
+
642
+
643
+ def check_endpoints_health(
644
+ container: Container,
645
+ docker_client: DockerClient,
646
+ container_ready: threading.Event,
647
+ stop_requested: threading.Event,
648
+ health_check: dict[str, str],
649
+ ) -> None:
650
+ """Continuously check /tokens and /v0/health endpoints"""
651
+ # Wait for container to be ready before starting health checks
652
+ container_ready.wait()
653
+
654
+ # Give container a moment to fully start up
655
+ time.sleep(2)
656
+
657
+ check_interval = 10 # Check every 10 seconds
658
+
659
+ while not stop_requested.is_set():
660
+ try:
661
+ # Check /tokens endpoint
662
+ tokens_response = requests.get(f"{TB_LOCAL_ADDRESS}/tokens", timeout=5)
663
+ if tokens_response.status_code != 200:
664
+ health_check["error"] = (
665
+ f"/tokens endpoint returned status {tokens_response.status_code}. Tinybird Local may be unhealthy."
666
+ )
667
+ stop_requested.set()
668
+ break
669
+
670
+ # Check /v0/health endpoint
671
+ health_response = requests.get(f"{TB_LOCAL_ADDRESS}/v0/health", timeout=5)
672
+ if health_response.status_code != 200:
673
+ health_check["error"] = (
674
+ f"/v0/health endpoint returned status {health_response.status_code}. "
675
+ "Tinybird Local may be unhealthy."
676
+ )
677
+ stop_requested.set()
678
+ break
679
+
680
+ # Verify tokens response has expected structure
681
+ try:
682
+ tokens_data = tokens_response.json()
683
+ if not all(key in tokens_data for key in ["user_token", "admin_token", "workspace_admin_token"]):
684
+ health_check["error"] = (
685
+ "/tokens endpoint returned unexpected data. Tinybird Local may be unhealthy."
686
+ )
687
+ stop_requested.set()
688
+ break
689
+ except json.JSONDecodeError:
690
+ health_check["error"] = "/tokens endpoint returned invalid JSON. Tinybird Local may be unhealthy."
691
+ stop_requested.set()
692
+ break
693
+
694
+ except Exception as e:
695
+ # Check if it's a connection error
696
+ error_str = str(e)
697
+ if "connect" in error_str.lower() or "timeout" in error_str.lower():
698
+ health_check["error"] = f"Failed to connect to Tinybird Local: {error_str}"
699
+ else:
700
+ health_check["error"] = f"Health check failed: {error_str}"
701
+ stop_requested.set()
702
+ break
703
+
704
+ if container_ready.is_set():
705
+ stats = container_stats(container, docker_client)
706
+ click.echo(f"{SERVICE_COLORS['[HEALTH]']}[HEALTH]{RESET} {stats}")
707
+
708
+ # Wait before next check
709
+ for _ in range(check_interval):
710
+ if stop_requested.is_set():
711
+ break
712
+ time.sleep(1)
713
+
714
+
715
+ def stream_logs_with_health_check(
716
+ container: Container, container_ready: threading.Event, stop_requested: threading.Event
717
+ ) -> None:
718
+ """Stream logs and monitor container health in parallel"""
719
+ # Wait for container to be ready before starting health checks
720
+ container_ready.wait()
721
+
722
+ # Give container a moment to fully start up
723
+ time.sleep(2)
724
+
725
+ retry_count = 0
726
+ max_retries = 10
727
+ exec_result = None
728
+
729
+ while retry_count < max_retries and not stop_requested.is_set():
730
+ try:
731
+ # Try to tail the log files (only new logs, not historical)
732
+ # Use -F to follow by name and retry if files don't exist yet
733
+ log_files = {
734
+ "/var/log/tinybird-local-server.log": "SERVER",
735
+ "/var/log/tinybird-local-hfi.log": "EVENTS",
736
+ "/var/log/tinybird-local-setup.log": "AUTH",
737
+ "/var/log/tinybird-local-kafka.log": "KAFKA",
738
+ }
739
+ # Build commands to tail each file and prefix with its label (using stdbuf for unbuffered output)
740
+ tail_commands = [
741
+ f'tail -n 0 -f {path} | stdbuf -oL sed "s/^/[{source}] /"' for path, source in log_files.items()
742
+ ]
743
+ # Join with & to run in parallel, then wait for all
744
+ cmd = f"sh -c '({' & '.join(tail_commands)}) & wait'"
745
+ exec_result = container.exec_run(cmd, stream=True, tty=False, stdout=True, stderr=True)
746
+ break # Success, exit retry loop
747
+ except Exception:
748
+ # Log file might not exist yet, wait and retry
749
+ retry_count += 1
750
+ if retry_count < max_retries:
751
+ time.sleep(2)
752
+
753
+ # Stream logs continuously
754
+ if exec_result:
755
+ try:
756
+ for line in exec_result.output:
757
+ if stop_requested.is_set():
758
+ break
759
+
760
+ raw_line = line.decode("utf-8").rstrip()
761
+ lines = raw_line.split("\n")
762
+
763
+ # Print "ready" message when container becomes healthy
764
+ if container_ready.is_set() and not hasattr(stream_logs_with_health_check, "ready_printed"):
765
+ click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
766
+ stream_logs_with_health_check.ready_printed = True # type: ignore
767
+
768
+ for line in lines:
769
+ # Apply color to service label
770
+ for service, color in SERVICE_COLORS.items():
771
+ if line.startswith(service):
772
+ message = line[len(service) :]
773
+ # extract content of message="...""
774
+ match = re.search(r'message="([^"]*)"', message)
775
+ if match:
776
+ message = match.group(1)
777
+ line = f"{color}{service}{RESET} {message}"
778
+ break
779
+
780
+ click.echo(line)
781
+
782
+ except Exception:
783
+ pass # Silently ignore errors when stream is interrupted