tinybird 0.0.1.dev165__tar.gz → 0.0.1.dev167__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of tinybird might be problematic. Click here for more details.

Files changed (111) hide show
  1. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/PKG-INFO +1 -1
  2. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/ch_utils/engine.py +5 -2
  3. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/__cli__.py +2 -2
  4. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/deployment.py +8 -2
  5. tinybird-0.0.1.dev167/tinybird/tb/modules/local_common.py +436 -0
  6. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/test.py +3 -10
  7. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird.egg-info/PKG-INFO +1 -1
  8. tinybird-0.0.1.dev165/tinybird/tb/modules/local_common.py +0 -123
  9. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/setup.cfg +0 -0
  10. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/__cli__.py +0 -0
  11. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/ch_utils/constants.py +0 -0
  12. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/check_pypi.py +0 -0
  13. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/client.py +0 -0
  14. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/config.py +0 -0
  15. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/connectors.py +0 -0
  16. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/context.py +0 -0
  17. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/datafile.py +0 -0
  18. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/datatypes.py +0 -0
  19. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/feedback_manager.py +0 -0
  20. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/git_settings.py +0 -0
  21. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/prompts.py +0 -0
  22. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/sql.py +0 -0
  23. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/sql_template.py +0 -0
  24. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/sql_template_fmt.py +0 -0
  25. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/sql_toolset.py +0 -0
  26. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/syncasync.py +0 -0
  27. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/check_pypi.py +0 -0
  28. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/cli.py +0 -0
  29. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/client.py +0 -0
  30. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/config.py +0 -0
  31. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/build.py +0 -0
  32. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/cicd.py +0 -0
  33. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/cli.py +0 -0
  34. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/common.py +0 -0
  35. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/config.py +0 -0
  36. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/connection.py +0 -0
  37. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/copy.py +0 -0
  38. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/create.py +0 -0
  39. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/build.py +0 -0
  40. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/build_common.py +0 -0
  41. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
  42. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/build_pipe.py +0 -0
  43. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/common.py +0 -0
  44. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/diff.py +0 -0
  45. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/exceptions.py +0 -0
  46. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/fixture.py +0 -0
  47. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/format_common.py +0 -0
  48. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
  49. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
  50. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/parse_datasource.py +0 -0
  51. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/parse_pipe.py +0 -0
  52. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
  53. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/playground.py +0 -0
  54. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datafile/pull.py +0 -0
  55. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/datasource.py +0 -0
  56. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/deprecations.py +0 -0
  57. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/dev_server.py +0 -0
  58. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/endpoint.py +0 -0
  59. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/exceptions.py +0 -0
  60. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/feedback_manager.py +0 -0
  61. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/info.py +0 -0
  62. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/infra.py +0 -0
  63. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/job.py +0 -0
  64. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/llm.py +0 -0
  65. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/llm_utils.py +0 -0
  66. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/local.py +0 -0
  67. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/login.py +0 -0
  68. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/logout.py +0 -0
  69. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/materialization.py +0 -0
  70. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/mock.py +0 -0
  71. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/open.py +0 -0
  72. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/pipe.py +0 -0
  73. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/project.py +0 -0
  74. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/regions.py +0 -0
  75. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/secret.py +0 -0
  76. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/shell.py +0 -0
  77. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/table.py +0 -0
  78. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/telemetry.py +0 -0
  79. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
  80. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
  81. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/token.py +0 -0
  82. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/watch.py +0 -0
  83. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/workspace.py +0 -0
  84. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb/modules/workspace_members.py +0 -0
  85. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli.py +0 -0
  86. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/auth.py +0 -0
  87. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/branch.py +0 -0
  88. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/cicd.py +0 -0
  89. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/cli.py +0 -0
  90. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/common.py +0 -0
  91. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/config.py +0 -0
  92. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/connection.py +0 -0
  93. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/datasource.py +0 -0
  94. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/exceptions.py +0 -0
  95. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/fmt.py +0 -0
  96. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/job.py +0 -0
  97. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/pipe.py +0 -0
  98. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/regions.py +0 -0
  99. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/tag.py +0 -0
  100. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/telemetry.py +0 -0
  101. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/test.py +0 -0
  102. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
  103. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
  104. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/workspace.py +0 -0
  105. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tb_cli_modules/workspace_members.py +0 -0
  106. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird/tornado_template.py +0 -0
  107. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird.egg-info/SOURCES.txt +0 -0
  108. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird.egg-info/dependency_links.txt +0 -0
  109. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird.egg-info/entry_points.txt +0 -0
  110. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird.egg-info/requires.txt +0 -0
  111. {tinybird-0.0.1.dev165 → tinybird-0.0.1.dev167}/tinybird.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev165
3
+ Version: 0.0.1.dev167
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -146,10 +146,9 @@ class TableDetails:
146
146
  def is_replacing_engine(self) -> bool:
147
147
  if self.engine:
148
148
  engine_lower = self.engine.lower()
149
- is_aggregating = "aggregatingmergetree" in engine_lower
150
149
  is_replacing = "replacingmergetree" in engine_lower
151
150
  is_collapsing = "collapsingmergetree" in engine_lower
152
- return is_aggregating or is_replacing or is_collapsing
151
+ return is_replacing or is_collapsing
153
152
  return False
154
153
 
155
154
  def diff_ttl(self, new_ttl: str) -> bool:
@@ -169,6 +168,10 @@ class TableDetails:
169
168
  @property
170
169
  def sorting_key(self) -> Optional[str]:
171
170
  _sorting_key = self.details.get("sorting_key", None)
171
+ # TODO: This should use ENABLED_ENGINES to guess if the sorting key is required or not
172
+ # Also checking this and raising an error in a getter is a bit of an anti-pattern,
173
+ # a data source could have a "wrong" sorting key and we won't be able to even show it in the API.
174
+ # All these checks be performed only on creation time.
172
175
  if self.is_replacing_engine() and not _sorting_key:
173
176
  raise ValueError(f"SORTING_KEY must be defined for the {self.engine} engine")
174
177
  if self.is_mergetree_family():
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
4
4
  __url__ = 'https://www.tinybird.co/docs/forward/commands'
5
5
  __author__ = 'Tinybird'
6
6
  __author_email__ = 'support@tinybird.co'
7
- __version__ = '0.0.1.dev165'
8
- __revision__ = '154a5d7'
7
+ __version__ = '0.0.1.dev167'
8
+ __revision__ = '1b40d30'
@@ -576,7 +576,10 @@ def create_deployment(
576
576
  sys.exit(0)
577
577
  elif status == "failed":
578
578
  click.echo(FeedbackManager.error(message="Deployment failed"))
579
- sys_exit("deployment_error", "Deployment failed" + str(deployment.get("errors")))
579
+ sys_exit(
580
+ "deployment_error",
581
+ f"Deployment failed. Errors: {str(deployment.get('errors') + deployment.get('feedback'))}",
582
+ )
580
583
  else:
581
584
  click.echo(FeedbackManager.error(message=f"Unknown deployment result {status}"))
582
585
  except Exception as e:
@@ -606,7 +609,10 @@ def create_deployment(
606
609
  if auto:
607
610
  click.echo(FeedbackManager.error(message="Rolling back deployment"))
608
611
  discard_deployment(client.host, HEADERS, wait=wait)
609
- sys_exit("deployment_error", "Deployment failed: " + str(deployment.get("errors")))
612
+ sys_exit(
613
+ "deployment_error",
614
+ f"Deployment failed. Errors: {str(deployment.get('errors') + deployment.get('feedback'))}",
615
+ )
610
616
 
611
617
  if deployment.get("status") == "data_ready":
612
618
  break
@@ -0,0 +1,436 @@
1
+ import hashlib
2
+ import json
3
+ import logging
4
+ import os
5
+ import re
6
+ import subprocess
7
+ import time
8
+ from typing import Any, Dict, Optional
9
+
10
+ import boto3
11
+ import click
12
+ import requests
13
+
14
+ import docker
15
+ from docker.client import DockerClient
16
+ from docker.models.containers import Container
17
+ from tinybird.tb.client import AuthNoTokenException, TinyB
18
+ from tinybird.tb.modules.config import CLIConfig
19
+ from tinybird.tb.modules.exceptions import CLILocalException
20
+ from tinybird.tb.modules.feedback_manager import FeedbackManager
21
+ from tinybird.tb.modules.telemetry import add_telemetry_event
22
+
23
+ TB_IMAGE_NAME = "tinybirdco/tinybird-local:latest"
24
+ TB_CONTAINER_NAME = "tinybird-local"
25
+ TB_LOCAL_PORT = int(os.getenv("TB_LOCAL_PORT", 7181))
26
+ TB_LOCAL_HOST = re.sub(r"^https?://", "", os.getenv("TB_LOCAL_HOST", "localhost"))
27
+ TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
28
+ TB_LOCAL_DEFAULT_WORKSPACE_NAME = "Tinybird_Local_Testing"
29
+
30
+
31
+ async def get_tinybird_local_client(
32
+ config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
33
+ ) -> TinyB:
34
+ """Get a Tinybird client connected to the local environment."""
35
+
36
+ config = await get_tinybird_local_config(config_obj, test=test, silent=silent)
37
+ return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
38
+
39
+
40
+ async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
41
+ """Craft a client config with a workspace name based on the path of the project files
42
+
43
+ It uses the tokens from tinybird local
44
+ """
45
+ path = config_obj.get("path")
46
+ config = CLIConfig.get_project_config()
47
+ tokens = get_local_tokens()
48
+ user_token = tokens["user_token"]
49
+ admin_token = tokens["admin_token"]
50
+ default_token = tokens["workspace_admin_token"]
51
+ # Create a new workspace if path is provided. This is used to isolate the build in a different workspace.
52
+ if path:
53
+ user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
54
+ if test:
55
+ ws_name = get_test_workspace_name(path)
56
+ else:
57
+ ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
58
+ if not ws_name:
59
+ raise AuthNoTokenException()
60
+
61
+ logging.debug(f"Workspace used for build: {ws_name}")
62
+
63
+ user_workspaces = requests.get(
64
+ f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?with_organization=true&token={admin_token}"
65
+ ).json()
66
+ user_org_id = user_workspaces.get("organization_id", {})
67
+ local_workspaces = user_workspaces.get("workspaces", [])
68
+
69
+ ws = next((ws for ws in local_workspaces if ws["name"] == ws_name), None)
70
+
71
+ # If we are running a test, we need to delete the workspace if it already exists
72
+ if test and ws:
73
+ requests.delete(
74
+ f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
75
+ )
76
+ ws = None
77
+
78
+ if not ws:
79
+ await user_client.create_workspace(ws_name, assign_to_organization_id=user_org_id, version="v1")
80
+ user_workspaces = requests.get(f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?token={admin_token}").json()
81
+ ws = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == ws_name), None)
82
+ if not ws:
83
+ raise AuthNoTokenException()
84
+
85
+ ws_token = ws["token"]
86
+ config.set_token(ws_token)
87
+ config.set_token_for_host(TB_LOCAL_ADDRESS, ws_token)
88
+ config.set_host(TB_LOCAL_ADDRESS)
89
+ else:
90
+ config.set_token(default_token)
91
+ config.set_token_for_host(TB_LOCAL_ADDRESS, default_token)
92
+
93
+ config.set_user_token(user_token)
94
+ return config
95
+
96
+
97
+ def get_build_workspace_name(path: str) -> str:
98
+ folder_hash = hashlib.sha256(path.encode()).hexdigest()
99
+ return f"Tinybird_Local_Build_{folder_hash}"
100
+
101
+
102
+ def get_test_workspace_name(path: str) -> str:
103
+ folder_hash = hashlib.sha256(path.encode()).hexdigest()
104
+ return f"Tinybird_Local_Test_{folder_hash}"
105
+
106
+
107
+ def get_local_tokens() -> Dict[str, str]:
108
+ try:
109
+ # ruff: noqa: ASYNC210
110
+ return requests.get(f"{TB_LOCAL_ADDRESS}/tokens").json()
111
+ except Exception:
112
+ # Check if tinybird-local is running using docker client (some clients use podman and won't have docker cmd)
113
+ try:
114
+ docker_client = get_docker_client()
115
+ container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, {})
116
+
117
+ output = {}
118
+ if container:
119
+ output = container.attrs
120
+ add_telemetry_event(
121
+ "docker_debug",
122
+ data={
123
+ "container_attrs": output,
124
+ },
125
+ )
126
+
127
+ if container and container.status == "running":
128
+ if container.health == "healthy":
129
+ raise CLILocalException(
130
+ FeedbackManager.error(
131
+ message=(
132
+ "Looks like Tinybird Local is running but we are not able to connect to it.\n\n"
133
+ "If you've run it manually using different host or port, please set the environment variables "
134
+ "TB_LOCAL_HOST and TB_LOCAL_PORT to match the ones you're using.\n"
135
+ "If you're not sure about this, please run `tb local restart` and try again."
136
+ )
137
+ )
138
+ )
139
+ raise CLILocalException(
140
+ FeedbackManager.error(
141
+ message=(
142
+ "Tinybird Local is running but it's unhealthy. Please check if it's running and try again.\n"
143
+ "If the problem persists, please run `tb local restart` and try again."
144
+ )
145
+ )
146
+ )
147
+ except CLILocalException as e:
148
+ raise e
149
+ except Exception:
150
+ pass
151
+
152
+ # Check if tinybird-local is running with docker
153
+ try:
154
+ output_str = subprocess.check_output(
155
+ ["docker", "ps", "--filter", f"name={TB_CONTAINER_NAME}", "--format", "json"], text=True
156
+ )
157
+ output = {}
158
+ if output_str:
159
+ output = json.loads(output_str)
160
+ add_telemetry_event(
161
+ "docker_debug",
162
+ data={
163
+ "docker_ps_output": output,
164
+ },
165
+ )
166
+
167
+ if output.get("State", "") == "running":
168
+ if "(healthy)" in output.get("Status", ""):
169
+ raise CLILocalException(
170
+ FeedbackManager.error(
171
+ message=(
172
+ "Looks like Tinybird Local is running but we are not able to connect to it.\n\n"
173
+ "If you've run it manually using different host or port, please set the environment variables "
174
+ "TB_LOCAL_HOST and TB_LOCAL_PORT to match the ones you're using.\n"
175
+ "If you're not sure about this, please run `tb local restart` and try again."
176
+ )
177
+ )
178
+ )
179
+ raise CLILocalException(
180
+ FeedbackManager.error(
181
+ message="Tinybird Local is running but it's unhealthy. Please check if it's running and try again.\n"
182
+ "If the problem persists, please run `tb local restart` and try again."
183
+ )
184
+ )
185
+ except CLILocalException as e:
186
+ raise e
187
+ except Exception:
188
+ pass
189
+
190
+ is_ci = (
191
+ os.getenv("GITHUB_ACTIONS")
192
+ or os.getenv("TRAVIS")
193
+ or os.getenv("CIRCLECI")
194
+ or os.getenv("GITLAB_CI")
195
+ or os.getenv("CI")
196
+ or os.getenv("TB_CI")
197
+ )
198
+ if not is_ci:
199
+ yes = click.confirm(
200
+ FeedbackManager.warning(message="Tinybird local is not running. Do you want to start it? [Y/n]"),
201
+ prompt_suffix="",
202
+ show_default=False,
203
+ default=True,
204
+ )
205
+ if yes:
206
+ click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
207
+ docker_client = get_docker_client()
208
+ start_tinybird_local(docker_client, False)
209
+ click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
210
+ return get_local_tokens()
211
+
212
+ raise CLILocalException(
213
+ FeedbackManager.error(message="Tinybird local is not running. Please run `tb local start` first.")
214
+ )
215
+
216
+
217
+ def start_tinybird_local(
218
+ docker_client: DockerClient,
219
+ use_aws_creds: bool,
220
+ ) -> None:
221
+ """Start the Tinybird container."""
222
+ pull_show_prompt = False
223
+ pull_required = False
224
+ try:
225
+ local_image = docker_client.images.get(TB_IMAGE_NAME)
226
+ local_image_id = local_image.attrs["RepoDigests"][0].split("@")[1]
227
+ remote_image = docker_client.images.get_registry_data(TB_IMAGE_NAME)
228
+ pull_show_prompt = local_image_id != remote_image.id
229
+ except Exception:
230
+ pull_show_prompt = False
231
+ pull_required = True
232
+
233
+ if pull_show_prompt and click.confirm(
234
+ FeedbackManager.warning(message="△ New version detected, download? [y/N]:"),
235
+ show_default=False,
236
+ prompt_suffix="",
237
+ ):
238
+ click.echo(FeedbackManager.info(message="* Downloading latest version of Tinybird Local..."))
239
+ pull_required = True
240
+
241
+ if pull_required:
242
+ docker_client.images.pull(TB_IMAGE_NAME, platform="linux/amd64")
243
+
244
+ environment = get_use_aws_creds() if use_aws_creds else {}
245
+
246
+ container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, environment)
247
+
248
+ if container and not pull_required:
249
+ # Container `start` is idempotent. It's safe to call it even if the container is already running.
250
+ container.start()
251
+ else:
252
+ if container:
253
+ container.remove(force=True)
254
+
255
+ container = docker_client.containers.run(
256
+ TB_IMAGE_NAME,
257
+ name=TB_CONTAINER_NAME,
258
+ detach=True,
259
+ ports={"7181/tcp": TB_LOCAL_PORT},
260
+ remove=False,
261
+ platform="linux/amd64",
262
+ environment=environment,
263
+ )
264
+
265
+ click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
266
+ while True:
267
+ container.reload() # Refresh container attributes
268
+ health = container.attrs.get("State", {}).get("Health", {}).get("Status")
269
+ if health == "healthy":
270
+ break
271
+ if health == "unhealthy":
272
+ raise CLILocalException(
273
+ FeedbackManager.error(
274
+ message="Tinybird Local is unhealthy. Try running `tb local restart` in a few seconds."
275
+ )
276
+ )
277
+
278
+ time.sleep(5)
279
+
280
+ # Remove tinybird-local dangling images to avoid running out of disk space
281
+ images = docker_client.images.list(name=re.sub(r":.*$", "", TB_IMAGE_NAME), all=True, filters={"dangling": True})
282
+ for image in images:
283
+ image.remove(force=True)
284
+
285
+
286
+ def get_existing_container_with_matching_env(
287
+ docker_client: DockerClient, container_name: str, required_env: dict[str, str]
288
+ ) -> Optional[Container]:
289
+ """
290
+ Checks if a container with the given name exists and has matching environment variables.
291
+ If it exists but environment doesn't match, it returns None.
292
+
293
+ Args:
294
+ docker_client: The Docker client instance
295
+ container_name: The name of the container to check
296
+ required_env: Dictionary of environment variables that must be present
297
+
298
+ Returns:
299
+ The container if it exists with matching environment, None otherwise
300
+ """
301
+ container = None
302
+ containers = docker_client.containers.list(all=True, filters={"name": container_name})
303
+ if containers:
304
+ container = containers[0]
305
+
306
+ if container and required_env:
307
+ container_info = container.attrs
308
+ container_env = container_info.get("Config", {}).get("Env", [])
309
+ env_missing = False
310
+ for key, value in required_env.items():
311
+ env_var = f"{key}={value}"
312
+ if env_var not in container_env:
313
+ env_missing = True
314
+ break
315
+
316
+ if env_missing:
317
+ container.remove(force=True)
318
+ container = None
319
+
320
+ return container
321
+
322
+
323
+ def get_docker_client() -> DockerClient:
324
+ """Check if Docker is installed and running."""
325
+ try:
326
+ docker_host = os.getenv("DOCKER_HOST")
327
+ if not docker_host:
328
+ # Try to get docker host from docker context
329
+ try:
330
+ try:
331
+ output = subprocess.check_output(["docker", "context", "inspect"], text=True)
332
+ except Exception as e:
333
+ add_telemetry_event(
334
+ "docker_error",
335
+ error=f"docker_context_inspect_error: {str(e)}",
336
+ )
337
+ raise e
338
+ try:
339
+ context = json.loads(output)
340
+ except Exception as e:
341
+ add_telemetry_event(
342
+ "docker_error",
343
+ error=f"docker_context_inspect_parse_output_error: {str(e)}",
344
+ data={
345
+ "docker_context_inspect_output": output,
346
+ },
347
+ )
348
+ raise e
349
+ if context and len(context) > 0:
350
+ try:
351
+ docker_host = context[0].get("Endpoints", {}).get("docker", {}).get("Host")
352
+ if docker_host:
353
+ os.environ["DOCKER_HOST"] = docker_host
354
+ except Exception as e:
355
+ add_telemetry_event(
356
+ "docker_error",
357
+ error=f"docker_context_parse_host_error: {str(e)}",
358
+ data={
359
+ "context": json.dumps(context),
360
+ },
361
+ )
362
+ raise e
363
+ except Exception:
364
+ pass
365
+ try:
366
+ client = docker.from_env() # type: ignore
367
+ except Exception as e:
368
+ add_telemetry_event(
369
+ "docker_error",
370
+ error=f"docker_get_client_from_env_error: {str(e)}",
371
+ )
372
+ raise e
373
+ try:
374
+ client.ping()
375
+ except Exception as e:
376
+ client_dict_non_sensitive = {k: v for k, v in client.api.__dict__.items() if "auth" not in k}
377
+ add_telemetry_event(
378
+ "docker_error",
379
+ error=f"docker_ping_error: {str(e)}",
380
+ data={
381
+ "client": repr(client_dict_non_sensitive),
382
+ },
383
+ )
384
+ raise e
385
+ return client
386
+ except Exception:
387
+ raise CLILocalException(
388
+ FeedbackManager.error(
389
+ message=(
390
+ f"No container runtime is running. Make sure a Docker-compatible runtime is installed and running. "
391
+ f"Trying to connect to Docker-compatible runtime at {docker_host}\n\n"
392
+ "If you're using a custom location, please provide it using the DOCKER_HOST environment variable."
393
+ )
394
+ )
395
+ )
396
+
397
+
398
+ def get_use_aws_creds() -> dict[str, str]:
399
+ credentials: dict[str, str] = {}
400
+ try:
401
+ # Get the boto3 session and credentials
402
+ session = boto3.Session()
403
+ creds = session.get_credentials()
404
+
405
+ if creds:
406
+ # Create environment variables for the container based on boto credentials
407
+ credentials["AWS_ACCESS_KEY_ID"] = creds.access_key
408
+ credentials["AWS_SECRET_ACCESS_KEY"] = creds.secret_key
409
+
410
+ # Add session token if it exists (for temporary credentials)
411
+ if creds.token:
412
+ credentials["AWS_SESSION_TOKEN"] = creds.token
413
+
414
+ # Add region if available
415
+ if session.region_name:
416
+ credentials["AWS_DEFAULT_REGION"] = session.region_name
417
+
418
+ click.echo(
419
+ FeedbackManager.success(
420
+ message=f"✓ AWS credentials found and will be passed to Tinybird Local (region: {session.region_name or 'not set'})"
421
+ )
422
+ )
423
+ else:
424
+ click.echo(
425
+ FeedbackManager.warning(
426
+ message="△ No AWS credentials found. S3 operations will not work in Tinybird Local."
427
+ )
428
+ )
429
+ except Exception as e:
430
+ click.echo(
431
+ FeedbackManager.warning(
432
+ message=f"△ Error retrieving AWS credentials: {str(e)}. S3 operations will not work in Tinybird Local."
433
+ )
434
+ )
435
+
436
+ return credentials
@@ -13,7 +13,6 @@ from pathlib import Path
13
13
  from typing import Any, Dict, List, Optional, Tuple
14
14
 
15
15
  import click
16
- import requests
17
16
  import yaml
18
17
  from requests import Response
19
18
 
@@ -22,11 +21,11 @@ from tinybird.tb.client import TinyB
22
21
  from tinybird.tb.modules.build import process as build_project
23
22
  from tinybird.tb.modules.cli import cli
24
23
  from tinybird.tb.modules.config import CLIConfig
25
- from tinybird.tb.modules.exceptions import CLILocalException, CLITestException
24
+ from tinybird.tb.modules.exceptions import CLITestException
26
25
  from tinybird.tb.modules.feedback_manager import FeedbackManager
27
26
  from tinybird.tb.modules.llm import LLM
28
27
  from tinybird.tb.modules.llm_utils import extract_xml, parse_xml
29
- from tinybird.tb.modules.local_common import TB_LOCAL_ADDRESS, get_test_workspace_name
28
+ from tinybird.tb.modules.local_common import get_local_tokens, get_test_workspace_name
30
29
  from tinybird.tb.modules.project import Project
31
30
 
32
31
  yaml.SafeDumper.org_represent_str = yaml.SafeDumper.represent_str # type: ignore[attr-defined]
@@ -314,13 +313,7 @@ def get_pipe_path(name_or_filename: str, folder: str) -> Path:
314
313
 
315
314
  def cleanup_test_workspace(client: TinyB, path: str) -> None:
316
315
  user_client = deepcopy(client)
317
- try:
318
- # ruff: noqa: ASYNC210
319
- tokens = requests.get(f"{TB_LOCAL_ADDRESS}/tokens").json()
320
- except Exception:
321
- raise CLILocalException(
322
- FeedbackManager.error(message="Tinybird local is not running. Please run `tb local start` first.")
323
- )
316
+ tokens = get_local_tokens()
324
317
  try:
325
318
  user_token = tokens["user_token"]
326
319
  user_client.token = user_token
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: tinybird
3
- Version: 0.0.1.dev165
3
+ Version: 0.0.1.dev167
4
4
  Summary: Tinybird Command Line Tool
5
5
  Home-page: https://www.tinybird.co/docs/forward/commands
6
6
  Author: Tinybird
@@ -1,123 +0,0 @@
1
- import hashlib
2
- import logging
3
- import os
4
- import re
5
- import subprocess
6
- from typing import Any, Dict
7
-
8
- import requests
9
-
10
- from tinybird.tb.client import AuthNoTokenException, TinyB
11
- from tinybird.tb.modules.config import CLIConfig
12
- from tinybird.tb.modules.exceptions import CLILocalException
13
- from tinybird.tb.modules.feedback_manager import FeedbackManager
14
- from tinybird.tb.modules.telemetry import add_telemetry_event
15
-
16
- TB_IMAGE_NAME = "tinybirdco/tinybird-local:latest"
17
- TB_CONTAINER_NAME = "tinybird-local"
18
- TB_LOCAL_PORT = int(os.getenv("TB_LOCAL_PORT", 7181))
19
- TB_LOCAL_HOST = re.sub(r"^https?://", "", os.getenv("TB_LOCAL_HOST", "localhost"))
20
- TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
21
- TB_LOCAL_DEFAULT_WORKSPACE_NAME = "Tinybird_Local_Testing"
22
-
23
-
24
- async def get_tinybird_local_client(
25
- config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
26
- ) -> TinyB:
27
- """Get a Tinybird client connected to the local environment."""
28
-
29
- config = await get_tinybird_local_config(config_obj, test=test, silent=silent)
30
- return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
31
-
32
-
33
- async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
34
- """Craft a client config with a workspace name based on the path of the project files
35
-
36
- It uses the tokens from tinybird local
37
- """
38
- path = config_obj.get("path")
39
- config = CLIConfig.get_project_config()
40
- tokens = get_local_tokens()
41
- user_token = tokens["user_token"]
42
- admin_token = tokens["admin_token"]
43
- default_token = tokens["workspace_admin_token"]
44
- # Create a new workspace if path is provided. This is used to isolate the build in a different workspace.
45
- if path:
46
- user_client = config.get_client(host=TB_LOCAL_ADDRESS, token=user_token)
47
- if test:
48
- ws_name = get_test_workspace_name(path)
49
- else:
50
- ws_name = config.get("name") or config_obj.get("name") or get_build_workspace_name(path)
51
- if not ws_name:
52
- raise AuthNoTokenException()
53
-
54
- logging.debug(f"Workspace used for build: {ws_name}")
55
-
56
- user_workspaces = requests.get(
57
- f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?with_organization=true&token={admin_token}"
58
- ).json()
59
- user_org_id = user_workspaces.get("organization_id", {})
60
- local_workspaces = user_workspaces.get("workspaces", [])
61
-
62
- ws = next((ws for ws in local_workspaces if ws["name"] == ws_name), None)
63
-
64
- # If we are running a test, we need to delete the workspace if it already exists
65
- if test and ws:
66
- requests.delete(
67
- f"{TB_LOCAL_ADDRESS}/v1/workspaces/{ws['id']}?token={user_token}&hard_delete_confirmation=yes"
68
- )
69
- ws = None
70
-
71
- if not ws:
72
- await user_client.create_workspace(ws_name, assign_to_organization_id=user_org_id, version="v1")
73
- user_workspaces = requests.get(f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?token={admin_token}").json()
74
- ws = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == ws_name), None)
75
- if not ws:
76
- raise AuthNoTokenException()
77
-
78
- ws_token = ws["token"]
79
- config.set_token(ws_token)
80
- config.set_token_for_host(TB_LOCAL_ADDRESS, ws_token)
81
- config.set_host(TB_LOCAL_ADDRESS)
82
- else:
83
- config.set_token(default_token)
84
- config.set_token_for_host(TB_LOCAL_ADDRESS, default_token)
85
-
86
- config.set_user_token(user_token)
87
- return config
88
-
89
-
90
- def get_build_workspace_name(path: str) -> str:
91
- folder_hash = hashlib.sha256(path.encode()).hexdigest()
92
- return f"Tinybird_Local_Build_{folder_hash}"
93
-
94
-
95
- def get_test_workspace_name(path: str) -> str:
96
- folder_hash = hashlib.sha256(path.encode()).hexdigest()
97
- return f"Tinybird_Local_Test_{folder_hash}"
98
-
99
-
100
- def get_local_tokens() -> Dict[str, str]:
101
- try:
102
- # ruff: noqa: ASYNC210
103
- return requests.get(f"{TB_LOCAL_ADDRESS}/tokens").json()
104
- except Exception:
105
- try:
106
- # Check if tinybird-local is running with docker, in case it's a config issue
107
- output = subprocess.check_output(["docker", "ps"], text=True)
108
- header_row = next((line for line in output.splitlines() if "CONTAINER ID" in line), "")
109
- tb_local_row = next(
110
- (line for line in output.splitlines() if TB_CONTAINER_NAME in line),
111
- f"{TB_CONTAINER_NAME} not found in output",
112
- )
113
- add_telemetry_event(
114
- "docker_debug",
115
- data={
116
- "docker_ps_output": header_row + tb_local_row,
117
- },
118
- )
119
- except Exception:
120
- pass
121
- raise CLILocalException(
122
- FeedbackManager.error(message="Tinybird local is not running. Please run `tb local start` first.")
123
- )