tinybird 4.5.11.dev0__py3-none-any.whl → 4.5.12.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/client.py +17 -3
- tinybird/tb/modules/branch.py +14 -5
- tinybird/tb/modules/cli.py +32 -3
- tinybird/tb/modules/common.py +47 -14
- tinybird/tb/modules/connection.py +23 -2
- tinybird/tb/modules/connection_kafka.py +429 -58
- tinybird/tb/modules/create.py +34 -11
- tinybird/tb/modules/datafile/build.py +32 -5
- tinybird/tb/modules/feedback_manager.py +3 -0
- tinybird/tb/modules/info.py +35 -11
- tinybird/tb/modules/preview.py +3 -2
- {tinybird-4.5.11.dev0.dist-info → tinybird-4.5.12.dev0.dist-info}/METADATA +14 -2
- {tinybird-4.5.11.dev0.dist-info → tinybird-4.5.12.dev0.dist-info}/RECORD +17 -17
- {tinybird-4.5.11.dev0.dist-info → tinybird-4.5.12.dev0.dist-info}/WHEEL +0 -0
- {tinybird-4.5.11.dev0.dist-info → tinybird-4.5.12.dev0.dist-info}/entry_points.txt +0 -0
- {tinybird-4.5.11.dev0.dist-info → tinybird-4.5.12.dev0.dist-info}/top_level.txt +0 -0
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '4.5.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '4.5.12.dev0'
|
|
8
|
+
__revision__ = '6a70694'
|
tinybird/tb/client.py
CHANGED
|
@@ -1010,16 +1010,29 @@ class TinyB:
|
|
|
1010
1010
|
kafka_sasl_mechanism="PLAIN",
|
|
1011
1011
|
kafka_security_protocol="SASL_SSL",
|
|
1012
1012
|
kafka_ssl_ca_pem=None,
|
|
1013
|
+
kafka_sasl_oauthbearer_method=None,
|
|
1014
|
+
kafka_sasl_oauthbearer_aws_region=None,
|
|
1015
|
+
kafka_sasl_oauthbearer_aws_role_arn=None,
|
|
1016
|
+
kafka_sasl_oauthbearer_aws_external_id=None,
|
|
1013
1017
|
):
|
|
1018
|
+
is_oauthbearer = kafka_sasl_mechanism == "OAUTHBEARER"
|
|
1014
1019
|
params = {
|
|
1015
1020
|
"service": "kafka",
|
|
1016
1021
|
"kafka_security_protocol": kafka_security_protocol,
|
|
1017
1022
|
"kafka_sasl_mechanism": kafka_sasl_mechanism,
|
|
1018
1023
|
"kafka_bootstrap_servers": kafka_bootstrap_servers,
|
|
1019
|
-
"kafka_sasl_plain_username": kafka_key,
|
|
1020
|
-
"kafka_sasl_plain_password": kafka_secret,
|
|
1021
1024
|
"name": kafka_connection_name,
|
|
1022
1025
|
}
|
|
1026
|
+
if is_oauthbearer:
|
|
1027
|
+
params["kafka_sasl_oauthbearer_method"] = kafka_sasl_oauthbearer_method
|
|
1028
|
+
params["kafka_sasl_oauthbearer_aws_region"] = kafka_sasl_oauthbearer_aws_region
|
|
1029
|
+
params["kafka_sasl_oauthbearer_aws_role_arn"] = kafka_sasl_oauthbearer_aws_role_arn
|
|
1030
|
+
# external_id is optional; the server fills it in from the workspace if absent.
|
|
1031
|
+
if kafka_sasl_oauthbearer_aws_external_id:
|
|
1032
|
+
params["kafka_sasl_oauthbearer_aws_external_id"] = kafka_sasl_oauthbearer_aws_external_id
|
|
1033
|
+
else:
|
|
1034
|
+
params["kafka_sasl_plain_username"] = kafka_key
|
|
1035
|
+
params["kafka_sasl_plain_password"] = kafka_secret
|
|
1023
1036
|
|
|
1024
1037
|
if kafka_schema_registry_url:
|
|
1025
1038
|
params["kafka_schema_registry_url"] = kafka_schema_registry_url
|
|
@@ -1185,7 +1198,8 @@ class TinyB:
|
|
|
1185
1198
|
def get_access_read_policy(self, service: str, bucket: Optional[str] = None) -> Dict[str, Any]:
|
|
1186
1199
|
params = {}
|
|
1187
1200
|
if bucket:
|
|
1188
|
-
|
|
1201
|
+
# The Kafka endpoint scopes the policy via `msk_cluster_arn`, not `bucket`.
|
|
1202
|
+
params["msk_cluster_arn" if service == "kafka" else "bucket"] = bucket
|
|
1189
1203
|
return self._req(f"/v0/integrations/{service}/policies/read-access-policy?{urlencode(params)}")
|
|
1190
1204
|
|
|
1191
1205
|
def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
tinybird/tb/modules/branch.py
CHANGED
|
@@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
|
|
10
|
-
from tinybird.tb.modules.cli import cli
|
|
10
|
+
from tinybird.tb.modules.cli import _looks_like_uuid, cli, ensure_valid_workspace_name
|
|
11
11
|
from tinybird.tb.modules.common import (
|
|
12
12
|
MAIN_BRANCH,
|
|
13
13
|
create_workspace_branch,
|
|
@@ -93,7 +93,8 @@ def branch_ls(sort: bool) -> None:
|
|
|
93
93
|
help="Wait for data branch jobs to finish, showing a progress bar. Disabled by default.",
|
|
94
94
|
)
|
|
95
95
|
def create_branch(branch_name: Optional[str], last_partition: bool, ignore_datasources: List[str], wait: bool) -> None:
|
|
96
|
-
|
|
96
|
+
normalized_branch_name = ensure_valid_workspace_name(branch_name) if branch_name else branch_name
|
|
97
|
+
create_workspace_branch(normalized_branch_name, last_partition, False, list(ignore_datasources), wait)
|
|
97
98
|
|
|
98
99
|
|
|
99
100
|
@branch.command(name="rm", short_help="Removes an branch from the workspace. It can't be recovered.")
|
|
@@ -194,11 +195,14 @@ def clear_branch(
|
|
|
194
195
|
if branch_name_or_id:
|
|
195
196
|
if branch_name_or_id == MAIN_BRANCH:
|
|
196
197
|
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
198
|
+
lookup_branch_name_or_id = (
|
|
199
|
+
branch_name_or_id if _looks_like_uuid(branch_name_or_id) else ensure_valid_workspace_name(branch_name_or_id)
|
|
200
|
+
)
|
|
197
201
|
workspace_to_clear = next(
|
|
198
202
|
(
|
|
199
203
|
workspace
|
|
200
204
|
for workspace in workspace_branches
|
|
201
|
-
if workspace["name"] ==
|
|
205
|
+
if workspace["name"] == lookup_branch_name_or_id or workspace["id"] == lookup_branch_name_or_id
|
|
202
206
|
),
|
|
203
207
|
None,
|
|
204
208
|
)
|
|
@@ -210,11 +214,16 @@ def clear_branch(
|
|
|
210
214
|
raise CLIBranchException(FeedbackManager.error_not_a_branch(cli=get_cli_name()))
|
|
211
215
|
|
|
212
216
|
if not workspace_to_clear:
|
|
213
|
-
raise CLIBranchException(
|
|
217
|
+
raise CLIBranchException(
|
|
218
|
+
FeedbackManager.error_branch(
|
|
219
|
+
branch=lookup_branch_name_or_id if branch_name_or_id else "",
|
|
220
|
+
cli=get_cli_name(),
|
|
221
|
+
)
|
|
222
|
+
)
|
|
214
223
|
|
|
215
224
|
branch_name = workspace_to_clear["name"]
|
|
216
225
|
if yes or click.confirm(FeedbackManager.warning_confirm_clear_workspace()):
|
|
217
|
-
was_current_branch = workspace_to_clear["id"] == config
|
|
226
|
+
was_current_branch = workspace_to_clear["id"] == config["id"]
|
|
218
227
|
client = config.get_client(token=current_main_workspace.get("token"))
|
|
219
228
|
try:
|
|
220
229
|
client.delete_branch(workspace_to_clear["id"])
|
tinybird/tb/modules/cli.py
CHANGED
|
@@ -68,6 +68,7 @@ PROJECT_TYPE_TYPESCRIPT = "ts-sdk"
|
|
|
68
68
|
PROJECT_TYPE_PYTHON = "python-sdk"
|
|
69
69
|
PROJECT_TYPE_CLI = "cli"
|
|
70
70
|
PROJECT_TYPES = {PROJECT_TYPE_TYPESCRIPT, PROJECT_TYPE_PYTHON, PROJECT_TYPE_CLI}
|
|
71
|
+
UUID_PATTERN = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE)
|
|
71
72
|
|
|
72
73
|
|
|
73
74
|
CLI_PROJECT_MARKERS = (
|
|
@@ -163,16 +164,44 @@ def is_main_git_branch(branch_name: Optional[str]) -> bool:
|
|
|
163
164
|
return branch_name in {"main", "master"}
|
|
164
165
|
|
|
165
166
|
|
|
166
|
-
def sanitize_branch_name(branch_name: str) -> str:
|
|
167
|
+
def sanitize_branch_name(branch_name: str, *, enforce_workspace_prefix_rules: bool = True) -> str:
|
|
167
168
|
sanitized = re.sub(r"[^a-zA-Z0-9_]", "_", branch_name)
|
|
168
169
|
sanitized = re.sub(r"_+", "_", sanitized)
|
|
169
|
-
|
|
170
|
+
sanitized = sanitized.strip("_")
|
|
171
|
+
if enforce_workspace_prefix_rules:
|
|
172
|
+
if sanitized and sanitized[0].isdigit():
|
|
173
|
+
sanitized = f"branch_{sanitized}"
|
|
174
|
+
if sanitized.startswith("d_"):
|
|
175
|
+
sanitized = f"branch_{sanitized}"
|
|
176
|
+
return sanitized
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _looks_like_uuid(value: str) -> bool:
|
|
180
|
+
return bool(UUID_PATTERN.fullmatch(value))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def ensure_valid_workspace_name(name: str, *, context: str = "Branch") -> str:
|
|
184
|
+
sanitized = sanitize_branch_name(name)
|
|
185
|
+
if not sanitized:
|
|
186
|
+
raise CLIException(
|
|
187
|
+
FeedbackManager.error(
|
|
188
|
+
message=(
|
|
189
|
+
f"{context} name '{name}' is not valid. "
|
|
190
|
+
"Name must start with a letter and contain only letters, numbers, and underscores."
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
if sanitized != name:
|
|
196
|
+
click.echo(FeedbackManager.warning_branch_name_sanitized(context=context, original=name, sanitized=sanitized))
|
|
197
|
+
|
|
198
|
+
return sanitized
|
|
170
199
|
|
|
171
200
|
|
|
172
201
|
def get_tinybird_branch_name_from_git_branch(branch_name: Optional[str]) -> Optional[str]:
|
|
173
202
|
if not branch_name:
|
|
174
203
|
return None
|
|
175
|
-
sanitized = sanitize_branch_name(branch_name)
|
|
204
|
+
sanitized = sanitize_branch_name(branch_name, enforce_workspace_prefix_rules=False)
|
|
176
205
|
return sanitized or None
|
|
177
206
|
|
|
178
207
|
|
tinybird/tb/modules/common.py
CHANGED
|
@@ -1105,26 +1105,57 @@ def is_url_valid(url):
|
|
|
1105
1105
|
return False
|
|
1106
1106
|
|
|
1107
1107
|
|
|
1108
|
-
def
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1108
|
+
def _parse_kafka_host_port(entry: str) -> tuple[str, int]:
|
|
1109
|
+
"""Parse a single Kafka bootstrap entry of the form 'host' or 'host:port'.
|
|
1110
|
+
|
|
1111
|
+
Raises CLIException on malformed input. Defaults the port to 9092 when omitted.
|
|
1112
|
+
"""
|
|
1113
|
+
parts = entry.split(":")
|
|
1112
1114
|
if len(parts) > 2:
|
|
1113
1115
|
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1114
1116
|
host = parts[0]
|
|
1115
1117
|
port_str = parts[1] if len(parts) == 2 else "9092"
|
|
1116
1118
|
try:
|
|
1117
|
-
|
|
1119
|
+
return host, int(port_str)
|
|
1118
1120
|
except Exception:
|
|
1119
1121
|
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def validate_kafka_bootstrap_servers(host_and_port):
|
|
1125
|
+
"""Validate a Kafka bootstrap-servers string (single entry or comma-separated list).
|
|
1126
|
+
|
|
1127
|
+
Format is checked for every entry. Connectivity is then probed per entry — Kafka
|
|
1128
|
+
only needs one reachable bootstrap broker to discover the rest of the cluster,
|
|
1129
|
+
so the validator passes as long as at least one host accepts a TCP connection.
|
|
1130
|
+
The last connection error is re-raised when every host is unreachable.
|
|
1131
|
+
"""
|
|
1132
|
+
if not isinstance(host_and_port, str):
|
|
1133
|
+
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1134
|
+
|
|
1135
|
+
entries = [e.strip() for e in host_and_port.split(",") if e.strip()]
|
|
1136
|
+
if not entries:
|
|
1137
|
+
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1138
|
+
|
|
1139
|
+
# Format-check every entry up front so the user hears about all bad ones at
|
|
1140
|
+
# once rather than discovering them sequentially.
|
|
1141
|
+
hosts = [_parse_kafka_host_port(e) for e in entries]
|
|
1142
|
+
|
|
1143
|
+
last_error: Optional[CLIException] = None
|
|
1144
|
+
for host, port in hosts:
|
|
1145
|
+
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
|
|
1146
|
+
try:
|
|
1147
|
+
sock.settimeout(3)
|
|
1148
|
+
sock.connect((host, port))
|
|
1149
|
+
return # at least one reachable broker is enough
|
|
1150
|
+
except TimeoutError:
|
|
1151
|
+
last_error = CLIException(
|
|
1152
|
+
FeedbackManager.error_kafka_bootstrap_server_conn_timeout(host=host, port=port)
|
|
1153
|
+
)
|
|
1154
|
+
except Exception:
|
|
1155
|
+
last_error = CLIException(FeedbackManager.error_kafka_bootstrap_server_conn(host=host, port=port))
|
|
1156
|
+
|
|
1157
|
+
if last_error is not None:
|
|
1158
|
+
raise last_error
|
|
1128
1159
|
|
|
1129
1160
|
|
|
1130
1161
|
def validate_kafka_key(s):
|
|
@@ -1137,9 +1168,11 @@ def validate_kafka_secret(s):
|
|
|
1137
1168
|
raise CLIException("Password format is not correct, it must be a string")
|
|
1138
1169
|
|
|
1139
1170
|
|
|
1140
|
-
def validate_string_connector_param(param, s):
|
|
1171
|
+
def validate_string_connector_param(param: str, s: str) -> None:
|
|
1141
1172
|
if not isinstance(s, str):
|
|
1142
1173
|
raise CLIConnectionException(param + " format is not correct, it must be a string")
|
|
1174
|
+
if not s or not s.strip():
|
|
1175
|
+
raise CLIConnectionException(param + " cannot be empty")
|
|
1143
1176
|
|
|
1144
1177
|
|
|
1145
1178
|
def validate_connection_name(client, connection_name, service):
|
|
@@ -40,6 +40,18 @@ from tinybird.tb.modules.feedback_manager import FeedbackManager, get_cli_name
|
|
|
40
40
|
from tinybird.tb.modules.project import Project
|
|
41
41
|
from tinybird.tb.modules.secret import save_secret_to_env_file
|
|
42
42
|
|
|
43
|
+
|
|
44
|
+
def _upper_or_none(_ctx: click.Context, _param: click.Parameter, value: Optional[str]) -> Optional[str]:
|
|
45
|
+
"""Click callback that uppercases the option value if present.
|
|
46
|
+
|
|
47
|
+
Kafka SASL mechanisms and security protocols are matched case-sensitively
|
|
48
|
+
downstream (CLI, client, server), so we normalize at the flag layer to
|
|
49
|
+
accept user input like `--sasl-mechanism oauthbearer` without silently
|
|
50
|
+
falling through the case-sensitive branches and breaking the request.
|
|
51
|
+
"""
|
|
52
|
+
return value.upper() if value else value
|
|
53
|
+
|
|
54
|
+
|
|
43
55
|
DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
44
56
|
DataConnectorType.KAFKA: [
|
|
45
57
|
"kafka_bootstrap_servers",
|
|
@@ -247,15 +259,22 @@ def connection_create_gcs(ctx: Context) -> None:
|
|
|
247
259
|
@click.option("--schema-registry-url", default=None, help="Avro Confluent Schema Registry URL")
|
|
248
260
|
@click.option(
|
|
249
261
|
"--sasl-mechanism",
|
|
250
|
-
default=
|
|
262
|
+
default=None,
|
|
263
|
+
callback=_upper_or_none,
|
|
251
264
|
help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
|
|
252
265
|
)
|
|
253
266
|
@click.option(
|
|
254
267
|
"--security-protocol",
|
|
255
|
-
default=
|
|
268
|
+
default=None,
|
|
269
|
+
callback=_upper_or_none,
|
|
256
270
|
help="Security protocol for connection-based protocols. Defaults to 'SASL_SSL'",
|
|
257
271
|
)
|
|
258
272
|
@click.option("--ssl-ca-pem", default=None, help="Path or content of the CA Certificate file in PEM format")
|
|
273
|
+
@click.option(
|
|
274
|
+
"--oauthbearer-aws-external-id",
|
|
275
|
+
default=None,
|
|
276
|
+
help="Pre-shared external_id for the AWS assume-role call (OAUTHBEARER+AWS only). If omitted, the wizard prompts for one and falls back to a Tinybird-generated value.",
|
|
277
|
+
)
|
|
259
278
|
@click.pass_context
|
|
260
279
|
def connection_create_kafka_cmd(
|
|
261
280
|
ctx: Context,
|
|
@@ -268,6 +287,7 @@ def connection_create_kafka_cmd(
|
|
|
268
287
|
sasl_mechanism: Optional[str],
|
|
269
288
|
security_protocol: Optional[str],
|
|
270
289
|
ssl_ca_pem: Optional[str],
|
|
290
|
+
oauthbearer_aws_external_id: Optional[str],
|
|
271
291
|
) -> None:
|
|
272
292
|
env: str = ctx.ensure_object(dict)["env"]
|
|
273
293
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
@@ -295,6 +315,7 @@ def connection_create_kafka_cmd(
|
|
|
295
315
|
sasl_mechanism=sasl_mechanism,
|
|
296
316
|
security_protocol=security_protocol,
|
|
297
317
|
ssl_ca_pem=ssl_ca_pem,
|
|
318
|
+
oauthbearer_aws_external_id=oauthbearer_aws_external_id,
|
|
298
319
|
)
|
|
299
320
|
|
|
300
321
|
if not result["error"]:
|
|
@@ -4,26 +4,289 @@
|
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
6
|
import re
|
|
7
|
+
import uuid
|
|
7
8
|
from datetime import datetime
|
|
8
|
-
from typing import Any, Dict, Optional
|
|
9
|
+
from typing import Any, Dict, Optional, Tuple
|
|
9
10
|
|
|
10
11
|
import click
|
|
12
|
+
import pyperclip
|
|
11
13
|
from click import Context
|
|
12
14
|
from confluent_kafka.admin import AdminClient
|
|
13
15
|
|
|
14
16
|
from tinybird.tb.client import TinyB
|
|
15
17
|
from tinybird.tb.modules.common import (
|
|
16
18
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
19
|
+
get_aws_iamrole_policies,
|
|
17
20
|
get_kafka_connection_name,
|
|
18
21
|
validate_kafka_bootstrap_servers,
|
|
22
|
+
validate_string_connector_param,
|
|
19
23
|
)
|
|
20
24
|
from tinybird.tb.modules.create import generate_kafka_connection_with_secrets
|
|
21
25
|
from tinybird.tb.modules.exceptions import CLIConnectionException, CLIException
|
|
22
26
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
27
|
+
from tinybird.tb.modules.local_common import get_tinybird_local_client
|
|
23
28
|
from tinybird.tb.modules.project import Project
|
|
24
29
|
from tinybird.tb.modules.secret import save_secret_to_env_file
|
|
25
30
|
from tinybird.tb.modules.telemetry import add_telemetry_event
|
|
26
31
|
|
|
32
|
+
# SASL mechanisms that authenticate with a username/password pair (the "key" + "secret"
|
|
33
|
+
# the wizard collects). OAUTHBEARER is intentionally excluded — its credentials come
|
|
34
|
+
# from the AWS IAM role flow, not from a key/secret prompt.
|
|
35
|
+
SASL_MECHANISMS_WITH_CREDENTIALS = ("PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def run_kafka_aws_iamrole_connection_flow(
|
|
39
|
+
config: Dict[str, Any],
|
|
40
|
+
client: TinyB,
|
|
41
|
+
connection_name: str,
|
|
42
|
+
external_id_override: Optional[str] = None,
|
|
43
|
+
) -> Tuple[str, str, str, str, Optional[TinyB], Optional[TinyB]]:
|
|
44
|
+
"""Interactive AWS IAM Role connection flow for Kafka (MSK).
|
|
45
|
+
|
|
46
|
+
Walks the user through creating an IAM access policy and role with the trust
|
|
47
|
+
policy that includes the AWS account IDs of the selected environments
|
|
48
|
+
(local, cloud, or both), then returns everything the caller needs to write
|
|
49
|
+
the `.connection` file and store the role ARN as a secret.
|
|
50
|
+
"""
|
|
51
|
+
service = "kafka"
|
|
52
|
+
|
|
53
|
+
msk_cluster_arn = click.prompt(
|
|
54
|
+
FeedbackManager.highlight(
|
|
55
|
+
message="? MSK Cluster ARN (e.g., arn:aws:kafka:us-east-1:123456789012:cluster/my-cluster/...)"
|
|
56
|
+
),
|
|
57
|
+
prompt_suffix="\n> ",
|
|
58
|
+
)
|
|
59
|
+
validate_string_connector_param("MSK Cluster ARN", msk_cluster_arn)
|
|
60
|
+
|
|
61
|
+
# ARN can be either ".../cluster/NAME" or ".../cluster/NAME/UUID"; cluster
|
|
62
|
+
# name is the second-to-last segment when a UUID is present, otherwise last.
|
|
63
|
+
if "/" in msk_cluster_arn:
|
|
64
|
+
arn_parts = msk_cluster_arn.split("/")
|
|
65
|
+
cluster_name = arn_parts[-2] if len(arn_parts) >= 3 else arn_parts[-1]
|
|
66
|
+
else:
|
|
67
|
+
cluster_name = "cluster"
|
|
68
|
+
|
|
69
|
+
try:
|
|
70
|
+
region = msk_cluster_arn.split(":")[3]
|
|
71
|
+
except (IndexError, AttributeError):
|
|
72
|
+
region = ""
|
|
73
|
+
|
|
74
|
+
if not region or not region.strip():
|
|
75
|
+
region = click.prompt(
|
|
76
|
+
FeedbackManager.highlight(message="? Region (the region where the MSK cluster is located)"),
|
|
77
|
+
default="us-east-1",
|
|
78
|
+
show_default=True,
|
|
79
|
+
prompt_suffix="\n> ",
|
|
80
|
+
)
|
|
81
|
+
validate_string_connector_param("Region", region)
|
|
82
|
+
|
|
83
|
+
cloud_client, local_client = _choose_environments_and_init_clients(config)
|
|
84
|
+
|
|
85
|
+
# Policy fetch can fail if the server doesn't have AWS credentials (typical for
|
|
86
|
+
# tb local) or if the role already exists out-of-band. We don't want that to
|
|
87
|
+
# abort the wizard — the user can still paste a known role ARN + external_id
|
|
88
|
+
# at the end. Show a warning and continue with placeholder text so the
|
|
89
|
+
# walkthrough steps below still display something sensible.
|
|
90
|
+
try:
|
|
91
|
+
access_policy, trust_policy, external_id = get_aws_iamrole_policies(
|
|
92
|
+
client,
|
|
93
|
+
service=service,
|
|
94
|
+
policy="read",
|
|
95
|
+
bucket=msk_cluster_arn,
|
|
96
|
+
external_id_seed=connection_name,
|
|
97
|
+
cloud_client=cloud_client,
|
|
98
|
+
local_client=local_client,
|
|
99
|
+
)
|
|
100
|
+
except Exception as e:
|
|
101
|
+
click.echo(
|
|
102
|
+
FeedbackManager.warning(
|
|
103
|
+
message=(
|
|
104
|
+
f"⚠ Could not auto-generate IAM policies from Tinybird ({e}). "
|
|
105
|
+
"Continuing anyway — you can still paste a pre-existing Role ARN + External ID below."
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
access_policy = "<could not generate — see your AWS admin or use an existing policy>"
|
|
110
|
+
trust_policy = "<could not generate — see your AWS admin or use an existing role>"
|
|
111
|
+
external_id = ""
|
|
112
|
+
|
|
113
|
+
click.echo(FeedbackManager.gray(message="\n» Step 1: AWS Authentication"))
|
|
114
|
+
click.echo(
|
|
115
|
+
FeedbackManager.info(
|
|
116
|
+
message="Please log into your AWS Console. We'll guide you through creating the necessary permissions: https://console.aws.amazon.com/"
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
click.echo(
|
|
120
|
+
FeedbackManager.info(
|
|
121
|
+
message="You'll be creating a single IAM Policy and Role to access your Kafka data. Using IAM Roles improves security by providing temporary credentials and following least privilege principles."
|
|
122
|
+
)
|
|
123
|
+
)
|
|
124
|
+
click.echo(FeedbackManager.click_enter_to_continue())
|
|
125
|
+
input()
|
|
126
|
+
|
|
127
|
+
access_policy_copied = False
|
|
128
|
+
try:
|
|
129
|
+
pyperclip.copy(access_policy)
|
|
130
|
+
access_policy_copied = True
|
|
131
|
+
except Exception:
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
click.echo(FeedbackManager.gray(message="» Step 2: Create IAM Policy"))
|
|
135
|
+
click.echo(
|
|
136
|
+
FeedbackManager.info(
|
|
137
|
+
message=f"1. Go to AWS IAM > Create Policy: https://console.aws.amazon.com/iamv2/home?region={region}#/policies/create"
|
|
138
|
+
)
|
|
139
|
+
)
|
|
140
|
+
click.echo(FeedbackManager.info(message="2. Select the JSON tab"))
|
|
141
|
+
if access_policy_copied:
|
|
142
|
+
click.echo(FeedbackManager.info(message="3. Paste the following policy (already copied to clipboard):"))
|
|
143
|
+
else:
|
|
144
|
+
click.echo(FeedbackManager.info(message="3. Copy and paste the following policy:"))
|
|
145
|
+
click.echo(FeedbackManager.highlight(message=f"\n{access_policy}\n"))
|
|
146
|
+
click.echo(
|
|
147
|
+
FeedbackManager.info(
|
|
148
|
+
message=f"4. Name the policy something meaningful (e.g., TinybirdKafkaAccess-{cluster_name})"
|
|
149
|
+
)
|
|
150
|
+
)
|
|
151
|
+
click.echo(FeedbackManager.info(message="5. Click 'Create policy'"))
|
|
152
|
+
click.echo(FeedbackManager.click_enter_to_continue())
|
|
153
|
+
input()
|
|
154
|
+
|
|
155
|
+
trust_policy_copied = False
|
|
156
|
+
try:
|
|
157
|
+
pyperclip.copy(trust_policy)
|
|
158
|
+
trust_policy_copied = True
|
|
159
|
+
except Exception:
|
|
160
|
+
pass
|
|
161
|
+
|
|
162
|
+
click.echo(FeedbackManager.gray(message="» Step 3: Create IAM Role"))
|
|
163
|
+
click.echo(
|
|
164
|
+
FeedbackManager.info(
|
|
165
|
+
message=f"1. Go to AWS IAM > Create Role: https://console.aws.amazon.com/iamv2/home?region={region}#/roles/create"
|
|
166
|
+
)
|
|
167
|
+
)
|
|
168
|
+
click.echo(FeedbackManager.info(message='2. Choose "Custom trust policy"'))
|
|
169
|
+
if trust_policy_copied:
|
|
170
|
+
click.echo(FeedbackManager.info(message="3. Paste the following trust policy (already copied to clipboard):"))
|
|
171
|
+
else:
|
|
172
|
+
click.echo(FeedbackManager.info(message="3. Paste the following trust policy:"))
|
|
173
|
+
click.echo(FeedbackManager.highlight(message=f"\n{trust_policy}\n"))
|
|
174
|
+
click.echo(FeedbackManager.info(message="4. Click Next, search for and select the policy you just created"))
|
|
175
|
+
click.echo(
|
|
176
|
+
FeedbackManager.info(message=f"5. Name the role something meaningful (e.g., TinybirdKafkaRole-{cluster_name})")
|
|
177
|
+
)
|
|
178
|
+
click.echo(FeedbackManager.info(message="6. Click 'Create role'"))
|
|
179
|
+
click.echo(FeedbackManager.info(message="7. Copy the Role ARN from the role details page"))
|
|
180
|
+
|
|
181
|
+
role_arn = click.prompt(
|
|
182
|
+
FeedbackManager.highlight(message="? Please enter the ARN of the role you just created"),
|
|
183
|
+
show_default=False,
|
|
184
|
+
)
|
|
185
|
+
validate_string_connector_param("Role ARN", role_arn)
|
|
186
|
+
|
|
187
|
+
# Allow a pre-shared external_id (e.g. when the role's trust policy was set up
|
|
188
|
+
# out-of-band with a specific External ID agreed between the cluster owner and
|
|
189
|
+
# Tinybird). Flag wins; otherwise prompt; empty answer keeps the server-generated one.
|
|
190
|
+
if external_id_override and external_id_override.strip():
|
|
191
|
+
external_id = external_id_override.strip()
|
|
192
|
+
else:
|
|
193
|
+
provided = click.prompt(
|
|
194
|
+
FeedbackManager.highlight(
|
|
195
|
+
message="? External ID (optional, leave blank to use the Tinybird-generated one shown in the trust policy above)"
|
|
196
|
+
),
|
|
197
|
+
default="",
|
|
198
|
+
show_default=False,
|
|
199
|
+
)
|
|
200
|
+
if provided and provided.strip():
|
|
201
|
+
external_id = provided.strip()
|
|
202
|
+
|
|
203
|
+
return role_arn, region, external_id, msk_cluster_arn, cloud_client, local_client
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _choose_environments_and_init_clients(
|
|
207
|
+
config: Dict[str, Any],
|
|
208
|
+
) -> Tuple[Optional[TinyB], Optional[TinyB]]:
|
|
209
|
+
"""Ask the user which environments the connection targets and initialize the
|
|
210
|
+
corresponding clients (used to create the role-ARN secret in both)."""
|
|
211
|
+
click.echo(
|
|
212
|
+
FeedbackManager.highlight(
|
|
213
|
+
message="? Which environments will use this connection? (the role-ARN secret will be created in the selected envs)"
|
|
214
|
+
)
|
|
215
|
+
)
|
|
216
|
+
click.echo(" [1] Local only")
|
|
217
|
+
click.echo(" [2] Cloud only")
|
|
218
|
+
click.echo(" [3] Both")
|
|
219
|
+
env_choice = click.prompt("\nSelect option", default=3, type=int)
|
|
220
|
+
|
|
221
|
+
if env_choice == 1:
|
|
222
|
+
use_local, use_cloud = True, False
|
|
223
|
+
elif env_choice == 2:
|
|
224
|
+
use_local, use_cloud = False, True
|
|
225
|
+
else:
|
|
226
|
+
if env_choice != 3:
|
|
227
|
+
click.echo(FeedbackManager.warning(message="Invalid option. Defaulting to 'Both'."))
|
|
228
|
+
use_local, use_cloud = True, True
|
|
229
|
+
|
|
230
|
+
local_client: Optional[TinyB] = None
|
|
231
|
+
cloud_client: Optional[TinyB] = None
|
|
232
|
+
|
|
233
|
+
if use_local:
|
|
234
|
+
try:
|
|
235
|
+
local_client, _ = get_tinybird_local_client(config)
|
|
236
|
+
except Exception as e:
|
|
237
|
+
click.echo(FeedbackManager.warning(message=f"Failed to initialize local client: {e}"))
|
|
238
|
+
|
|
239
|
+
if use_cloud:
|
|
240
|
+
try:
|
|
241
|
+
cloud_client = TinyB(token=config.get("token", ""), host=config.get("host", ""), staging=False)
|
|
242
|
+
except Exception as e:
|
|
243
|
+
click.echo(FeedbackManager.warning(message=f"Failed to initialize cloud client: {e}"))
|
|
244
|
+
|
|
245
|
+
return cloud_client, local_client
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def run_kafka_aws_iamrole_existing_role_flow(
|
|
249
|
+
config: Dict[str, Any],
|
|
250
|
+
external_id_override: Optional[str] = None,
|
|
251
|
+
) -> Tuple[str, str, str, str, Optional[TinyB], Optional[TinyB]]:
|
|
252
|
+
"""Fast path for users who already have an IAM role configured for MSK.
|
|
253
|
+
|
|
254
|
+
Skips the policy fetch and the AWS Console walkthrough entirely. Collects
|
|
255
|
+
region + role ARN + external ID + env choice for secret storage.
|
|
256
|
+
"""
|
|
257
|
+
region = click.prompt(
|
|
258
|
+
FeedbackManager.highlight(message="? AWS region of the MSK cluster"),
|
|
259
|
+
default="us-east-1",
|
|
260
|
+
show_default=True,
|
|
261
|
+
)
|
|
262
|
+
validate_string_connector_param("Region", region)
|
|
263
|
+
|
|
264
|
+
role_arn = click.prompt(
|
|
265
|
+
FeedbackManager.highlight(message="? IAM Role ARN to assume for MSK"),
|
|
266
|
+
show_default=False,
|
|
267
|
+
)
|
|
268
|
+
validate_string_connector_param("Role ARN", role_arn)
|
|
269
|
+
|
|
270
|
+
external_id = ""
|
|
271
|
+
if external_id_override and external_id_override.strip():
|
|
272
|
+
external_id = external_id_override.strip()
|
|
273
|
+
else:
|
|
274
|
+
provided = click.prompt(
|
|
275
|
+
FeedbackManager.highlight(
|
|
276
|
+
message="? External ID (leave blank to let Tinybird derive one from the workspace)"
|
|
277
|
+
),
|
|
278
|
+
default="",
|
|
279
|
+
show_default=False,
|
|
280
|
+
)
|
|
281
|
+
if provided and provided.strip():
|
|
282
|
+
external_id = provided.strip()
|
|
283
|
+
|
|
284
|
+
cloud_client, local_client = _choose_environments_and_init_clients(config)
|
|
285
|
+
|
|
286
|
+
# msk_cluster_arn is never written to the .connection file — return "" since
|
|
287
|
+
# the caller ignores it on this code path.
|
|
288
|
+
return role_arn, region, external_id, "", cloud_client, local_client
|
|
289
|
+
|
|
27
290
|
|
|
28
291
|
def connection_create_kafka(
|
|
29
292
|
ctx: Context,
|
|
@@ -36,10 +299,12 @@ def connection_create_kafka(
|
|
|
36
299
|
sasl_mechanism: Optional[str] = None,
|
|
37
300
|
security_protocol: Optional[str] = None,
|
|
38
301
|
ssl_ca_pem: Optional[str] = None,
|
|
302
|
+
oauthbearer_aws_external_id: Optional[str] = None,
|
|
39
303
|
) -> dict[str, Any]:
|
|
40
304
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
41
305
|
click.echo(FeedbackManager.gray(message="\n» Creating Kafka connection..."))
|
|
42
306
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
307
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
43
308
|
name = get_kafka_connection_name(project.folder, connection_name)
|
|
44
309
|
error: Optional[str] = None
|
|
45
310
|
|
|
@@ -81,40 +346,6 @@ def connection_create_kafka(
|
|
|
81
346
|
except Exception as e:
|
|
82
347
|
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
83
348
|
|
|
84
|
-
key = click.prompt(FeedbackManager.highlight(message="? Kafka key"))
|
|
85
|
-
|
|
86
|
-
assert isinstance(key, str)
|
|
87
|
-
|
|
88
|
-
secret_required = click.confirm(
|
|
89
|
-
FeedbackManager.info(message=" ? Do you want to store the Kafka key in a .env.local file? [Y/n]"),
|
|
90
|
-
default=True,
|
|
91
|
-
show_default=False,
|
|
92
|
-
)
|
|
93
|
-
|
|
94
|
-
if secret_required:
|
|
95
|
-
tb_secret_key = str(click.prompt(FeedbackManager.info(message=" ? Secret name")))
|
|
96
|
-
try:
|
|
97
|
-
save_secret_to_env_file(project=project, name=tb_secret_key, value=key)
|
|
98
|
-
except Exception as e:
|
|
99
|
-
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
100
|
-
|
|
101
|
-
secret = secret or click.prompt(FeedbackManager.highlight(message="? Kafka secret"), hide_input=True)
|
|
102
|
-
|
|
103
|
-
assert isinstance(secret, str)
|
|
104
|
-
|
|
105
|
-
secret_required = click.confirm(
|
|
106
|
-
FeedbackManager.info(message=" ? Do you want to store the Kafka secret in a .env.local file? [Y/n]"),
|
|
107
|
-
default=True,
|
|
108
|
-
show_default=False,
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
if secret_required:
|
|
112
|
-
tb_secret_secret = str(click.prompt(FeedbackManager.info(message=" ? Secret name")))
|
|
113
|
-
try:
|
|
114
|
-
save_secret_to_env_file(project=project, name=tb_secret_secret, value=secret)
|
|
115
|
-
except Exception as e:
|
|
116
|
-
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
117
|
-
|
|
118
349
|
security_protocol_options = ["SASL_SSL", "SASL_PLAINTEXT", "PLAINTEXT"]
|
|
119
350
|
security_protocol = security_protocol or click.prompt(
|
|
120
351
|
FeedbackManager.highlight(message="? Security Protocol (SASL_SSL, SASL_PLAINTEXT, PLAINTEXT) [SASL_SSL]"),
|
|
@@ -127,17 +358,132 @@ def connection_create_kafka(
|
|
|
127
358
|
if security_protocol not in security_protocol_options:
|
|
128
359
|
raise CLIConnectionException(FeedbackManager.error(message=f"Invalid security protocol: {security_protocol}"))
|
|
129
360
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
361
|
+
kafka_sasl_oauthbearer_method: Optional[str] = None
|
|
362
|
+
kafka_sasl_oauthbearer_aws_region: Optional[str] = None
|
|
363
|
+
kafka_sasl_oauthbearer_aws_role_arn: Optional[str] = None
|
|
364
|
+
kafka_sasl_oauthbearer_aws_external_id: Optional[str] = None
|
|
365
|
+
tb_secret_aws_role_arn: Optional[str] = None
|
|
366
|
+
# Track if the role-ARN secret was already created in cloud during the OAUTHBEARER
|
|
367
|
+
# flow so we don't prompt the user a second time in the cloud-secrets block below.
|
|
368
|
+
aws_role_arn_secret_created_in_cloud = False
|
|
369
|
+
|
|
370
|
+
# PLAINTEXT doesn't use SASL, so skip the mechanism prompt entirely.
|
|
371
|
+
if security_protocol == "PLAINTEXT":
|
|
372
|
+
sasl_mechanism = None
|
|
373
|
+
else:
|
|
374
|
+
sasl_mechanism_options = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512", "OAUTHBEARER"]
|
|
375
|
+
sasl_mechanism = sasl_mechanism or click.prompt(
|
|
376
|
+
FeedbackManager.highlight(
|
|
377
|
+
message="? SASL Mechanism (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512, OAUTHBEARER) [PLAIN]"
|
|
378
|
+
),
|
|
379
|
+
type=click.Choice(sasl_mechanism_options),
|
|
380
|
+
show_default=False,
|
|
381
|
+
show_choices=False,
|
|
382
|
+
default="PLAIN",
|
|
383
|
+
)
|
|
384
|
+
if sasl_mechanism not in sasl_mechanism_options:
|
|
385
|
+
raise CLIConnectionException(FeedbackManager.error(message=f"Invalid SASL mechanism: {sasl_mechanism}"))
|
|
138
386
|
|
|
139
|
-
if sasl_mechanism
|
|
140
|
-
|
|
387
|
+
if sasl_mechanism == "OAUTHBEARER":
|
|
388
|
+
kafka_sasl_oauthbearer_method = "AWS"
|
|
389
|
+
|
|
390
|
+
# Fast-path for users who already have the IAM role + trust policy set up
|
|
391
|
+
# (common: the role is owned by the same team as the MSK cluster and the
|
|
392
|
+
# external_id was pre-shared). Skips policy fetch + AWS Console walkthrough.
|
|
393
|
+
has_existing_role = click.confirm(
|
|
394
|
+
FeedbackManager.highlight(
|
|
395
|
+
message="? Do you already have an IAM role configured for this MSK cluster? [y/N]"
|
|
396
|
+
),
|
|
397
|
+
default=False,
|
|
398
|
+
show_default=False,
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
if has_existing_role:
|
|
402
|
+
(
|
|
403
|
+
kafka_sasl_oauthbearer_aws_role_arn,
|
|
404
|
+
kafka_sasl_oauthbearer_aws_region,
|
|
405
|
+
kafka_sasl_oauthbearer_aws_external_id,
|
|
406
|
+
_,
|
|
407
|
+
cloud_client,
|
|
408
|
+
local_client,
|
|
409
|
+
) = run_kafka_aws_iamrole_existing_role_flow(
|
|
410
|
+
config=obj["config"],
|
|
411
|
+
external_id_override=oauthbearer_aws_external_id,
|
|
412
|
+
)
|
|
413
|
+
else:
|
|
414
|
+
(
|
|
415
|
+
kafka_sasl_oauthbearer_aws_role_arn,
|
|
416
|
+
kafka_sasl_oauthbearer_aws_region,
|
|
417
|
+
kafka_sasl_oauthbearer_aws_external_id,
|
|
418
|
+
_,
|
|
419
|
+
cloud_client,
|
|
420
|
+
local_client,
|
|
421
|
+
) = run_kafka_aws_iamrole_connection_flow(
|
|
422
|
+
config=obj["config"],
|
|
423
|
+
client=client,
|
|
424
|
+
connection_name=name,
|
|
425
|
+
external_id_override=oauthbearer_aws_external_id,
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
# Auto-store the role ARN as a secret (in local + cloud per the user's choice
|
|
429
|
+
# of environments) so the .connection file can reference it via tb_secret().
|
|
430
|
+
unique_suffix = uuid.uuid4().hex[:8]
|
|
431
|
+
secret_name = f"kafka_role_arn_{name}_{unique_suffix}"
|
|
432
|
+
secret_created = False
|
|
433
|
+
|
|
434
|
+
if local_client and kafka_sasl_oauthbearer_aws_role_arn:
|
|
435
|
+
try:
|
|
436
|
+
save_secret_to_env_file(project=project, name=secret_name, value=kafka_sasl_oauthbearer_aws_role_arn)
|
|
437
|
+
secret_created = True
|
|
438
|
+
except Exception as e:
|
|
439
|
+
click.echo(FeedbackManager.warning(message=f"Failed to create secret in local: {e}"))
|
|
440
|
+
|
|
441
|
+
if cloud_client and kafka_sasl_oauthbearer_aws_role_arn:
|
|
442
|
+
try:
|
|
443
|
+
cloud_client.create_secret(name=secret_name, value=kafka_sasl_oauthbearer_aws_role_arn)
|
|
444
|
+
secret_created = True
|
|
445
|
+
aws_role_arn_secret_created_in_cloud = True
|
|
446
|
+
except Exception as e:
|
|
447
|
+
click.echo(FeedbackManager.warning(message=f"Failed to create secret in cloud: {e}"))
|
|
448
|
+
|
|
449
|
+
if secret_created:
|
|
450
|
+
tb_secret_aws_role_arn = secret_name
|
|
451
|
+
else:
|
|
452
|
+
click.echo(
|
|
453
|
+
FeedbackManager.warning(
|
|
454
|
+
message="No secrets were created. The role ARN will be stored directly in the connection file."
|
|
455
|
+
)
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
# PLAIN/SCRAM still need a username + password.
|
|
459
|
+
if sasl_mechanism in SASL_MECHANISMS_WITH_CREDENTIALS:
|
|
460
|
+
key = key or click.prompt(FeedbackManager.highlight(message="? Kafka key"))
|
|
461
|
+
assert isinstance(key, str)
|
|
462
|
+
|
|
463
|
+
if click.confirm(
|
|
464
|
+
FeedbackManager.info(message=" ? Do you want to store the Kafka key in a .env.local file? [Y/n]"),
|
|
465
|
+
default=True,
|
|
466
|
+
show_default=False,
|
|
467
|
+
):
|
|
468
|
+
tb_secret_key = str(click.prompt(FeedbackManager.info(message=" ? Secret name")))
|
|
469
|
+
try:
|
|
470
|
+
save_secret_to_env_file(project=project, name=tb_secret_key, value=key)
|
|
471
|
+
except Exception as e:
|
|
472
|
+
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
473
|
+
|
|
474
|
+
secret = secret or click.prompt(FeedbackManager.highlight(message="? Kafka secret"), hide_input=True)
|
|
475
|
+
assert isinstance(secret, str)
|
|
476
|
+
|
|
477
|
+
if click.confirm(
|
|
478
|
+
FeedbackManager.info(message=" ? Do you want to store the Kafka secret in a .env.local file? [Y/n]"),
|
|
479
|
+
default=True,
|
|
480
|
+
show_default=False,
|
|
481
|
+
):
|
|
482
|
+
tb_secret_secret = str(click.prompt(FeedbackManager.info(message=" ? Secret name")))
|
|
483
|
+
try:
|
|
484
|
+
save_secret_to_env_file(project=project, name=tb_secret_secret, value=secret)
|
|
485
|
+
except Exception as e:
|
|
486
|
+
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
141
487
|
|
|
142
488
|
if not schema_registry_url:
|
|
143
489
|
schema_registry_url = click.prompt(
|
|
@@ -168,6 +514,15 @@ def connection_create_kafka(
|
|
|
168
514
|
except Exception as e:
|
|
169
515
|
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
170
516
|
|
|
517
|
+
# Skip the role-ARN secret in this check if it was already created in cloud
|
|
518
|
+
# by the OAUTHBEARER flow above.
|
|
519
|
+
has_secrets_needing_cloud_creation = (
|
|
520
|
+
tb_secret_bootstrap_servers
|
|
521
|
+
or tb_secret_key
|
|
522
|
+
or tb_secret_secret
|
|
523
|
+
or tb_secret_ssl_ca_pem
|
|
524
|
+
or (tb_secret_aws_role_arn and not aws_role_arn_secret_created_in_cloud)
|
|
525
|
+
)
|
|
171
526
|
create_in_cloud = (
|
|
172
527
|
click.confirm(
|
|
173
528
|
FeedbackManager.highlight(
|
|
@@ -176,8 +531,7 @@ def connection_create_kafka(
|
|
|
176
531
|
default=True,
|
|
177
532
|
show_default=False,
|
|
178
533
|
)
|
|
179
|
-
if obj["env"] == "local"
|
|
180
|
-
and (tb_secret_bootstrap_servers or tb_secret_key or tb_secret_secret or tb_secret_ssl_ca_pem)
|
|
534
|
+
if obj["env"] == "local" and has_secrets_needing_cloud_creation
|
|
181
535
|
else False
|
|
182
536
|
)
|
|
183
537
|
|
|
@@ -194,24 +548,36 @@ def connection_create_kafka(
|
|
|
194
548
|
)
|
|
195
549
|
if tb_secret_bootstrap_servers:
|
|
196
550
|
prod_client.create_secret(name=tb_secret_bootstrap_servers, value=bootstrap_servers)
|
|
197
|
-
|
|
551
|
+
# tb_secret_key/tb_secret_secret are only set in the PLAIN/SCRAM branch,
|
|
552
|
+
# where key/secret are guaranteed strings.
|
|
553
|
+
if tb_secret_key and key is not None:
|
|
198
554
|
prod_client.create_secret(name=tb_secret_key, value=key)
|
|
199
|
-
if tb_secret_secret:
|
|
555
|
+
if tb_secret_secret and secret is not None:
|
|
200
556
|
prod_client.create_secret(name=tb_secret_secret, value=secret)
|
|
201
557
|
if tb_secret_ssl_ca_pem and ssl_ca_pem:
|
|
202
558
|
prod_client.create_secret(name=tb_secret_ssl_ca_pem, value=ssl_ca_pem)
|
|
559
|
+
if tb_secret_aws_role_arn and kafka_sasl_oauthbearer_aws_role_arn and not aws_role_arn_secret_created_in_cloud:
|
|
560
|
+
prod_client.create_secret(name=tb_secret_aws_role_arn, value=kafka_sasl_oauthbearer_aws_role_arn)
|
|
203
561
|
click.echo(FeedbackManager.success(message="✓ Secrets created!"))
|
|
204
562
|
|
|
205
|
-
click.echo(FeedbackManager.gray(message="» Validating connection..."))
|
|
206
|
-
|
|
207
563
|
topics: list[str] = []
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
564
|
+
if sasl_mechanism in SASL_MECHANISMS_WITH_CREDENTIALS:
|
|
565
|
+
click.echo(FeedbackManager.gray(message="» Validating connection..."))
|
|
566
|
+
try:
|
|
567
|
+
assert key is not None and secret is not None
|
|
568
|
+
topics = list_kafka_topics(bootstrap_servers, key, secret, security_protocol, sasl_mechanism, ssl_ca_pem)
|
|
569
|
+
click.echo(FeedbackManager.success(message="✓ Connection is valid"))
|
|
570
|
+
except Exception as e:
|
|
571
|
+
error = str(e)
|
|
572
|
+
click.echo(FeedbackManager.error(message=f"Connection is not valid: {e}"))
|
|
573
|
+
add_telemetry_event("connection_error", error=error)
|
|
574
|
+
else:
|
|
575
|
+
# OAUTHBEARER (no AWS creds locally) and PLAINTEXT defer validation to deploy-time.
|
|
576
|
+
click.echo(
|
|
577
|
+
FeedbackManager.info(
|
|
578
|
+
message=f"⚠ Skipping local validation for {sasl_mechanism or 'PLAINTEXT'}. The connection will be validated on deploy."
|
|
579
|
+
)
|
|
580
|
+
)
|
|
215
581
|
|
|
216
582
|
generate_kafka_connection_with_secrets(
|
|
217
583
|
name=name,
|
|
@@ -226,6 +592,11 @@ def connection_create_kafka(
|
|
|
226
592
|
ssl_ca_pem=ssl_ca_pem,
|
|
227
593
|
tb_secret_ssl_ca_pem=tb_secret_ssl_ca_pem,
|
|
228
594
|
schema_registry_url=schema_registry_url,
|
|
595
|
+
kafka_sasl_oauthbearer_method=kafka_sasl_oauthbearer_method,
|
|
596
|
+
kafka_sasl_oauthbearer_aws_region=kafka_sasl_oauthbearer_aws_region,
|
|
597
|
+
kafka_sasl_oauthbearer_aws_role_arn=kafka_sasl_oauthbearer_aws_role_arn,
|
|
598
|
+
kafka_sasl_oauthbearer_aws_external_id=kafka_sasl_oauthbearer_aws_external_id,
|
|
599
|
+
tb_secret_aws_role_arn=tb_secret_aws_role_arn,
|
|
229
600
|
folder=project.folder,
|
|
230
601
|
)
|
|
231
602
|
click.echo(FeedbackManager.info_file_created(file=f"connections/{name}.connection"))
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -561,35 +561,58 @@ def persist_tinybird_config(root_folder: str, project_type: str, dev_mode: str,
|
|
|
561
561
|
def generate_kafka_connection_with_secrets(
|
|
562
562
|
name: str,
|
|
563
563
|
bootstrap_servers: str,
|
|
564
|
-
key: str,
|
|
565
|
-
secret: str,
|
|
564
|
+
key: Optional[str],
|
|
565
|
+
secret: Optional[str],
|
|
566
566
|
tb_secret_bootstrap_servers: Optional[str],
|
|
567
567
|
tb_secret_key: Optional[str],
|
|
568
568
|
tb_secret_secret: Optional[str],
|
|
569
569
|
security_protocol: str,
|
|
570
|
-
sasl_mechanism: str,
|
|
570
|
+
sasl_mechanism: Optional[str],
|
|
571
571
|
ssl_ca_pem: Optional[str],
|
|
572
572
|
tb_secret_ssl_ca_pem: Optional[str],
|
|
573
573
|
schema_registry_url: Optional[str],
|
|
574
574
|
folder: str,
|
|
575
|
+
kafka_sasl_oauthbearer_method: Optional[str] = None,
|
|
576
|
+
kafka_sasl_oauthbearer_aws_region: Optional[str] = None,
|
|
577
|
+
kafka_sasl_oauthbearer_aws_role_arn: Optional[str] = None,
|
|
578
|
+
kafka_sasl_oauthbearer_aws_external_id: Optional[str] = None,
|
|
579
|
+
tb_secret_aws_role_arn: Optional[str] = None,
|
|
575
580
|
) -> Path:
|
|
576
581
|
kafka_bootstrap_servers = (
|
|
577
582
|
inject_tb_secret(tb_secret_bootstrap_servers) if tb_secret_bootstrap_servers else bootstrap_servers
|
|
578
583
|
)
|
|
579
|
-
kafka_key = inject_tb_secret(tb_secret_key) if tb_secret_key else key
|
|
580
|
-
kafka_secret = inject_tb_secret(tb_secret_secret) if tb_secret_secret else secret
|
|
581
|
-
kafka_ssl_ca_pem = inject_tb_secret(tb_secret_ssl_ca_pem) if tb_secret_ssl_ca_pem else ssl_ca_pem
|
|
582
584
|
content = f"""TYPE kafka
|
|
583
585
|
KAFKA_BOOTSTRAP_SERVERS {kafka_bootstrap_servers}
|
|
584
586
|
KAFKA_SECURITY_PROTOCOL {security_protocol or "SASL_SSL"}
|
|
585
|
-
KAFKA_SASL_MECHANISM {sasl_mechanism or "PLAIN"}
|
|
586
|
-
KAFKA_KEY {kafka_key}
|
|
587
|
-
KAFKA_SECRET {kafka_secret}
|
|
588
587
|
"""
|
|
588
|
+
# PLAINTEXT skips SASL entirely.
|
|
589
|
+
if sasl_mechanism:
|
|
590
|
+
content += f"KAFKA_SASL_MECHANISM {sasl_mechanism}\n"
|
|
591
|
+
|
|
592
|
+
if sasl_mechanism == "OAUTHBEARER" and kafka_sasl_oauthbearer_method == "AWS":
|
|
593
|
+
content += f"KAFKA_SASL_OAUTHBEARER_METHOD {kafka_sasl_oauthbearer_method}\n"
|
|
594
|
+
if kafka_sasl_oauthbearer_aws_region:
|
|
595
|
+
content += f"KAFKA_SASL_OAUTHBEARER_AWS_REGION {kafka_sasl_oauthbearer_aws_region}\n"
|
|
596
|
+
if kafka_sasl_oauthbearer_aws_role_arn:
|
|
597
|
+
kafka_role_arn = (
|
|
598
|
+
inject_tb_secret(tb_secret_aws_role_arn)
|
|
599
|
+
if tb_secret_aws_role_arn
|
|
600
|
+
else kafka_sasl_oauthbearer_aws_role_arn
|
|
601
|
+
)
|
|
602
|
+
content += f"KAFKA_SASL_OAUTHBEARER_AWS_ROLE_ARN {kafka_role_arn}\n"
|
|
603
|
+
if kafka_sasl_oauthbearer_aws_external_id:
|
|
604
|
+
content += f"KAFKA_SASL_OAUTHBEARER_AWS_EXTERNAL_ID {kafka_sasl_oauthbearer_aws_external_id}\n"
|
|
605
|
+
elif sasl_mechanism in ("PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"):
|
|
606
|
+
kafka_key = inject_tb_secret(tb_secret_key) if tb_secret_key else key
|
|
607
|
+
kafka_secret = inject_tb_secret(tb_secret_secret) if tb_secret_secret else secret
|
|
608
|
+
content += f"KAFKA_KEY {kafka_key}\nKAFKA_SECRET {kafka_secret}\n"
|
|
609
|
+
|
|
589
610
|
if schema_registry_url:
|
|
590
|
-
content += f"
|
|
611
|
+
content += f"KAFKA_SCHEMA_REGISTRY_URL {schema_registry_url}\n"
|
|
612
|
+
|
|
613
|
+
kafka_ssl_ca_pem = inject_tb_secret(tb_secret_ssl_ca_pem) if tb_secret_ssl_ca_pem else ssl_ca_pem
|
|
591
614
|
if kafka_ssl_ca_pem:
|
|
592
|
-
content += f"
|
|
615
|
+
content += f"KAFKA_SSL_CA_PEM >\n {kafka_ssl_ca_pem}\n"
|
|
593
616
|
content += """# Learn more at https://www.tinybird.co/docs/forward/get-data-in/connectors/kafka#kafka-connection-settings
|
|
594
617
|
"""
|
|
595
618
|
|
|
@@ -816,6 +816,10 @@ def process_file(
|
|
|
816
816
|
|
|
817
817
|
if not skip_connectors:
|
|
818
818
|
try:
|
|
819
|
+
is_oauthbearer = params.get("kafka_sasl_mechanism") == "OAUTHBEARER"
|
|
820
|
+
# PLAINTEXT brokers have no SASL at all, so a key/secret pair is
|
|
821
|
+
# neither produced by the wizard nor expected on the wire.
|
|
822
|
+
is_plaintext = params.get("kafka_security_protocol") == "PLAINTEXT"
|
|
819
823
|
connector_params = {
|
|
820
824
|
"kafka_bootstrap_servers": params.get("kafka_bootstrap_servers", None),
|
|
821
825
|
"kafka_key": params.get("kafka_key", None),
|
|
@@ -825,6 +829,16 @@ def process_file(
|
|
|
825
829
|
"kafka_schema_registry_url": params.get("kafka_schema_registry_url", None),
|
|
826
830
|
"kafka_ssl_ca_pem": get_ca_pem_content(params.get("kafka_ssl_ca_pem", None), filename),
|
|
827
831
|
"kafka_sasl_mechanism": params.get("kafka_sasl_mechanism", None),
|
|
832
|
+
# Explicit default — None would be dropped by the client's
|
|
833
|
+
# `if value is not None` filter, leaving the server to pick
|
|
834
|
+
# its own protocol instead of the SASL_SSL we want.
|
|
835
|
+
"kafka_security_protocol": params.get("kafka_security_protocol", "SASL_SSL"),
|
|
836
|
+
"kafka_sasl_oauthbearer_method": params.get("kafka_sasl_oauthbearer_method", None),
|
|
837
|
+
"kafka_sasl_oauthbearer_aws_region": params.get("kafka_sasl_oauthbearer_aws_region", None),
|
|
838
|
+
"kafka_sasl_oauthbearer_aws_role_arn": params.get("kafka_sasl_oauthbearer_aws_role_arn", None),
|
|
839
|
+
"kafka_sasl_oauthbearer_aws_external_id": params.get(
|
|
840
|
+
"kafka_sasl_oauthbearer_aws_external_id", None
|
|
841
|
+
),
|
|
828
842
|
}
|
|
829
843
|
|
|
830
844
|
connector = tb_client.get_connection(**connector_params)
|
|
@@ -832,11 +846,24 @@ def process_file(
|
|
|
832
846
|
click.echo(
|
|
833
847
|
FeedbackManager.info_creating_kafka_connection(connection_name=params["kafka_connection_name"])
|
|
834
848
|
)
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
849
|
+
if is_oauthbearer:
|
|
850
|
+
required_params = [
|
|
851
|
+
connector_params["kafka_bootstrap_servers"],
|
|
852
|
+
connector_params["kafka_sasl_oauthbearer_method"],
|
|
853
|
+
connector_params["kafka_sasl_oauthbearer_aws_region"],
|
|
854
|
+
connector_params["kafka_sasl_oauthbearer_aws_role_arn"],
|
|
855
|
+
]
|
|
856
|
+
elif is_plaintext:
|
|
857
|
+
# No SASL, no credentials — only the broker address matters.
|
|
858
|
+
required_params = [
|
|
859
|
+
connector_params["kafka_bootstrap_servers"],
|
|
860
|
+
]
|
|
861
|
+
else:
|
|
862
|
+
required_params = [
|
|
863
|
+
connector_params["kafka_bootstrap_servers"],
|
|
864
|
+
connector_params["kafka_key"],
|
|
865
|
+
connector_params["kafka_secret"],
|
|
866
|
+
]
|
|
840
867
|
|
|
841
868
|
if not all(required_params):
|
|
842
869
|
raise click.ClickException(
|
|
@@ -729,6 +729,9 @@ STEP 3: ADD KEY TO SERVICE ACCOUNT
|
|
|
729
729
|
)
|
|
730
730
|
warning_name_already_exists = warning_message("** Warning: {name} already exists, skipping")
|
|
731
731
|
warning_dry_name_already_exists = warning_message("** [DRY RUN] {name} already exists, skipping")
|
|
732
|
+
warning_branch_name_sanitized = warning_message(
|
|
733
|
+
"** Warning: {context} name '{original}' is not valid; using '{sanitized}' instead."
|
|
734
|
+
)
|
|
732
735
|
warning_datasource_cannot_be_pushed = warning_message(
|
|
733
736
|
"** [DRY RUN] Failed pushing datasource. Top level object names must be unique. {name} cannot have same name as an existing pipe."
|
|
734
737
|
)
|
tinybird/tb/modules/info.py
CHANGED
|
@@ -22,6 +22,7 @@ def info(ctx: click.Context, skip_local: bool) -> None:
|
|
|
22
22
|
ctx_config = ctx.ensure_object(dict)["config"]
|
|
23
23
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
24
24
|
output = ctx.ensure_object(dict)["output"]
|
|
25
|
+
selected_branch = ctx.ensure_object(dict).get("branch")
|
|
25
26
|
|
|
26
27
|
if output not in {"human", "json"}:
|
|
27
28
|
force_echo(FeedbackManager.error_invalid_output_format(formats=", ".join(["human", "json"])))
|
|
@@ -43,7 +44,7 @@ def info(ctx: click.Context, skip_local: bool) -> None:
|
|
|
43
44
|
click.echo(FeedbackManager.highlight(message="\n» Project:"))
|
|
44
45
|
project_table, project_columns = get_project_info(project.folder)
|
|
45
46
|
|
|
46
|
-
branches_summary = get_branches_summary(ctx_config)
|
|
47
|
+
branches_summary = get_branches_summary(ctx_config, selected_branch=selected_branch)
|
|
47
48
|
|
|
48
49
|
if output == "human":
|
|
49
50
|
click.echo(FeedbackManager.highlight(message="\n» Branches:"))
|
|
@@ -75,11 +76,14 @@ def info(ctx: click.Context, skip_local: bool) -> None:
|
|
|
75
76
|
project_data = {column: project_table[0][i] for i, column in enumerate(project_columns)}
|
|
76
77
|
response["project"] = project_data
|
|
77
78
|
|
|
78
|
-
|
|
79
|
+
branches_data: dict[str, Any] = {
|
|
79
80
|
"current": branches_summary["current"],
|
|
80
81
|
"count": branches_summary["count"],
|
|
81
82
|
"items": branches_summary["items"],
|
|
82
83
|
}
|
|
84
|
+
if branches_summary.get("error"):
|
|
85
|
+
branches_data["error"] = branches_summary["error"]
|
|
86
|
+
response["branches"] = branches_data
|
|
83
87
|
|
|
84
88
|
echo_json(response)
|
|
85
89
|
|
|
@@ -216,7 +220,7 @@ def get_project_info(project_path: Optional[str] = None) -> Tuple[Iterable[Any],
|
|
|
216
220
|
return table, columns
|
|
217
221
|
|
|
218
222
|
|
|
219
|
-
def get_branches_summary(ctx_config: Dict[str, Any]) -> Dict[str, Any]:
|
|
223
|
+
def get_branches_summary(ctx_config: Dict[str, Any], selected_branch: Optional[str] = None) -> Dict[str, Any]:
|
|
220
224
|
columns = ["name", "current", "token", "ui"]
|
|
221
225
|
|
|
222
226
|
try:
|
|
@@ -235,30 +239,39 @@ def get_branches_summary(ctx_config: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
235
239
|
current_workspace = client.workspace_info(version="v1")
|
|
236
240
|
|
|
237
241
|
main_workspace_name = current_workspace.get("name", "No workspace")
|
|
242
|
+
main_workspace_id = current_workspace.get("id")
|
|
243
|
+
main_workspace_token = current_workspace.get("token")
|
|
238
244
|
if current_workspace.get("is_branch"):
|
|
239
245
|
main_workspace_id = current_workspace.get("main")
|
|
240
246
|
main_workspace = next((ws for ws in workspaces if ws.get("id") == main_workspace_id), None)
|
|
241
247
|
if main_workspace and main_workspace.get("name"):
|
|
242
248
|
main_workspace_name = main_workspace["name"]
|
|
249
|
+
main_workspace_token = main_workspace.get("token") or main_workspace_token
|
|
243
250
|
|
|
244
251
|
branches = client.branches().get("environments", [])
|
|
252
|
+
if not branches and api_host and main_workspace_token:
|
|
253
|
+
fallback_client = TinyB(main_workspace_token, host=api_host)
|
|
254
|
+
branches = fallback_client.branches().get("environments", [])
|
|
245
255
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
256
|
+
if main_workspace_id:
|
|
257
|
+
scoped_branches = [branch for branch in branches if branch.get("main") == main_workspace_id]
|
|
258
|
+
if scoped_branches:
|
|
259
|
+
branches = scoped_branches
|
|
249
260
|
|
|
250
|
-
|
|
261
|
+
all_items = []
|
|
251
262
|
for branch in branches:
|
|
252
263
|
branch_name = branch.get("name") or "No branch name"
|
|
253
264
|
branch_token = branch.get("token") or "No token found"
|
|
254
|
-
branch_current =
|
|
265
|
+
branch_current = (
|
|
266
|
+
branch_name == selected_branch if selected_branch else current_workspace.get("id") == branch.get("id")
|
|
267
|
+
)
|
|
255
268
|
branch_ui = (
|
|
256
269
|
f"{ui_host}/{main_workspace_name}~{branch_name}"
|
|
257
270
|
if ui_host and main_workspace_name and branch_name != "No branch name"
|
|
258
271
|
else "No UI URL found"
|
|
259
272
|
)
|
|
260
273
|
|
|
261
|
-
|
|
274
|
+
all_items.append(
|
|
262
275
|
{
|
|
263
276
|
"name": branch_name,
|
|
264
277
|
"current": branch_current,
|
|
@@ -267,16 +280,27 @@ def get_branches_summary(ctx_config: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
267
280
|
}
|
|
268
281
|
)
|
|
269
282
|
|
|
283
|
+
items = all_items
|
|
284
|
+
error: Optional[str] = None
|
|
285
|
+
if selected_branch:
|
|
286
|
+
items = [branch for branch in all_items if branch["name"] == selected_branch]
|
|
287
|
+
if not items:
|
|
288
|
+
error = f"Branch '{selected_branch}' not found."
|
|
289
|
+
|
|
290
|
+
current_branch = selected_branch or "main"
|
|
291
|
+
if not selected_branch and current_workspace.get("is_branch"):
|
|
292
|
+
current_branch = current_workspace.get("name", "main")
|
|
293
|
+
|
|
270
294
|
items.sort(key=lambda branch: (not branch["current"], branch["name"]))
|
|
271
295
|
table = [(branch["name"], branch["current"], branch["token"], branch["ui"]) for branch in items]
|
|
272
296
|
|
|
273
297
|
return {
|
|
274
298
|
"current": current_branch,
|
|
275
|
-
"count": len(
|
|
299
|
+
"count": len(all_items),
|
|
276
300
|
"items": items,
|
|
277
301
|
"table": table,
|
|
278
302
|
"columns": columns,
|
|
279
|
-
"error":
|
|
303
|
+
"error": error,
|
|
280
304
|
}
|
|
281
305
|
except Exception:
|
|
282
306
|
return {
|
tinybird/tb/modules/preview.py
CHANGED
|
@@ -6,6 +6,7 @@ from click.core import ParameterSource
|
|
|
6
6
|
from tinybird.tb.client import TinyB
|
|
7
7
|
from tinybird.tb.modules.cli import (
|
|
8
8
|
cli,
|
|
9
|
+
ensure_valid_workspace_name,
|
|
9
10
|
get_current_git_branch,
|
|
10
11
|
sanitize_branch_name,
|
|
11
12
|
)
|
|
@@ -16,7 +17,7 @@ from tinybird.tb.modules.project import Project
|
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
def generate_preview_branch_name(git_branch: Optional[str]) -> str:
|
|
19
|
-
branch_part = sanitize_branch_name(git_branch or "")
|
|
20
|
+
branch_part = sanitize_branch_name(git_branch or "", enforce_workspace_prefix_rules=False)
|
|
20
21
|
return f"tmp_ci_{branch_part or 'unknown'}"
|
|
21
22
|
|
|
22
23
|
|
|
@@ -113,7 +114,7 @@ def preview(ctx: click.Context, dry_run: bool, check: bool, name: Optional[str])
|
|
|
113
114
|
output: str = obj.get("output", "human")
|
|
114
115
|
|
|
115
116
|
git_branch = get_current_git_branch()
|
|
116
|
-
preview_name = name
|
|
117
|
+
preview_name = ensure_valid_workspace_name(name) if name else generate_preview_branch_name(git_branch)
|
|
117
118
|
|
|
118
119
|
if dry_run:
|
|
119
120
|
click.echo(FeedbackManager.info(message=f"[dry-run] Preview target '{preview_name}' (cloud)"))
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 4.5.
|
|
3
|
+
Version: 4.5.12.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/forward/commands
|
|
6
6
|
Author: Tinybird
|
|
@@ -52,10 +52,22 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
52
52
|
Changelog
|
|
53
53
|
----------
|
|
54
54
|
|
|
55
|
-
4.5.
|
|
55
|
+
4.5.12
|
|
56
56
|
*******
|
|
57
57
|
|
|
58
|
+
- `Fixed` `tb info` now lists branches and marks the current branch correctly when running on a branch.
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
4.5.11
|
|
62
|
+
*******
|
|
63
|
+
|
|
64
|
+
- `Added` `tb connection create kafka` now supports SASL ``OAUTHBEARER`` for Amazon MSK clusters with IAM-based authentication. The wizard asks up-front whether the user already has an IAM role for the cluster — if yes, it collects region + role ARN + external ID directly; if no, it walks the user through creating the IAM access policy and trust policy in the AWS Console, copying them to the clipboard.
|
|
65
|
+
- `Added` ``--oauthbearer-aws-external-id`` flag (and matching interactive prompt) so users can supply a pre-shared external ID instead of the one Tinybird auto-generates for the connection.
|
|
66
|
+
- `Added` ``PLAINTEXT`` security protocol now skips the SASL mechanism prompt entirely.
|
|
67
|
+
- `Added` ``.connection`` files generated by the wizard now emit ``KAFKA_SASL_OAUTHBEARER_METHOD``, ``KAFKA_SASL_OAUTHBEARER_AWS_REGION``, ``KAFKA_SASL_OAUTHBEARER_AWS_ROLE_ARN``, and ``KAFKA_SASL_OAUTHBEARER_AWS_EXTERNAL_ID``, with the role ARN stored as a ``tb_secret``.
|
|
68
|
+
- `Fixed` ``tb connection create kafka`` now accepts comma-separated bootstrap-server lists (e.g. ``b-1.kafka:9092,b-2.kafka:9092``) instead of rejecting them as malformed. Validation passes as soon as any one broker is reachable, matching Kafka bootstrap semantics.
|
|
58
69
|
- `Fixed` `tb init` now persists `folder` in `tinybird.config.json`, so subsequent resource creation targets the configured project folder instead of the repository root.
|
|
70
|
+
- `Fixed` Branch names derived from Git refs are now sanitized in `tb preview --name` and `tb branch create/rm/clear`, preventing CI/CD failures with invalid workspace names.
|
|
59
71
|
|
|
60
72
|
4.5.9
|
|
61
73
|
*******
|
|
@@ -17,32 +17,32 @@ tinybird/datafile/exceptions.py,sha256=8rw2umdZjtby85QbuRKFO5ETz_eRHwUY5l7eHsy1w
|
|
|
17
17
|
tinybird/datafile/parse_connection.py,sha256=GxmGp_XnWbDZPDbh_PBxitlIMqZRYfDwxMBw-JQBp1g,1890
|
|
18
18
|
tinybird/datafile/parse_datasource.py,sha256=yd58HrUF4yNJXLn6OsvKGpZJpvrcjLGAeJG1lgBe_zk,1891
|
|
19
19
|
tinybird/datafile/parse_pipe.py,sha256=-9bbgVuiWRyDYydrLVflDBt8GstZotMy6dklsrc6MUY,3859
|
|
20
|
-
tinybird/tb/__cli__.py,sha256=
|
|
20
|
+
tinybird/tb/__cli__.py,sha256=2w-3VoyDce1czZ4H8qw7uO_Z7c589_0ZwLs7nXauCjA,246
|
|
21
21
|
tinybird/tb/check_pypi.py,sha256=Gp0HkHHDFMSDL6nxKlOY51z7z1Uv-2LRexNTZSHHGmM,552
|
|
22
22
|
tinybird/tb/cli.py,sha256=IjiGfNIpxSxi1odK1kMj9s8lEhx3sAUgGA263XdmyR0,1119
|
|
23
|
-
tinybird/tb/client.py,sha256=
|
|
23
|
+
tinybird/tb/client.py,sha256=reEc7efMfEgv4aZFizIZ2iGsOeTo8D2CKeOOBidI7fk,54883
|
|
24
24
|
tinybird/tb/config.py,sha256=CGCfBbCMlmVcBHQ0IMGc2IE4O2-1tZEPyD564JZoTbw,5659
|
|
25
|
-
tinybird/tb/modules/branch.py,sha256=
|
|
25
|
+
tinybird/tb/modules/branch.py,sha256=QxcMPc_4xhw8WPz85YZ5L6SQZY-lZrF3CleujBEBJQU,9746
|
|
26
26
|
tinybird/tb/modules/build.py,sha256=bGFoFppR_UbGUMDWFGDzfJ4nT3CGFwzCzl2o4OpwR2o,10420
|
|
27
27
|
tinybird/tb/modules/build_common.py,sha256=dxNRfvLcgJ6AwU4hsNDPaQT8QwirVCJzw4VLZaBoImM,23867
|
|
28
28
|
tinybird/tb/modules/cicd.py,sha256=IO4qqsoLRXcubALb7vx_QnRpg3zIIxfaVO9bGomlESY,8267
|
|
29
|
-
tinybird/tb/modules/cli.py,sha256=
|
|
30
|
-
tinybird/tb/modules/common.py,sha256=
|
|
29
|
+
tinybird/tb/modules/cli.py,sha256=n_B9EapB1eYOWOfRHN1Jefip6LcmYrkLDIz5GoGCJhA,41889
|
|
30
|
+
tinybird/tb/modules/common.py,sha256=AoibuekIqBJ6IW3tlp4uzxb5Q3upqhjOlqjxXR-wFxc,93337
|
|
31
31
|
tinybird/tb/modules/config.py,sha256=Z47lMZxFeX68b62bTzgj9379zn-9eT4cPbrcMJ_xTGQ,11431
|
|
32
|
-
tinybird/tb/modules/connection.py,sha256=
|
|
33
|
-
tinybird/tb/modules/connection_kafka.py,sha256=
|
|
32
|
+
tinybird/tb/modules/connection.py,sha256=sPR9PAZNSZnbFqsBKos9Fmr8PsAGIuk57bjZU5mhUsA,16290
|
|
33
|
+
tinybird/tb/modules/connection_kafka.py,sha256=Qc8uUNl5SvUcUGb_aG3AirYKw6673FoX1Sfdlzgz6Y8,33530
|
|
34
34
|
tinybird/tb/modules/connection_s3.py,sha256=-Sj9d7xAJXbpr8JHioCqrO2hneJ1spoBE4QpYEfiyS8,16565
|
|
35
35
|
tinybird/tb/modules/copy.py,sha256=Apzjxiyinp6KmgamypPKEe3BAnpxG7MwYcSYkHiG8sA,6033
|
|
36
|
-
tinybird/tb/modules/create.py,sha256=
|
|
36
|
+
tinybird/tb/modules/create.py,sha256=KpqqVznYOU0OZfsxzZsMAWKoFULo0pJRiOSaACdtRoE,22506
|
|
37
37
|
tinybird/tb/modules/datasource.py,sha256=52P3qz4Su9OCNOyXy5JnbkN3miMgxsfJ5AdJiJ9nFAI,64455
|
|
38
38
|
tinybird/tb/modules/deployment.py,sha256=TNDlvaYmk-zouvaAdMXlyFPbgbnMKQOt76xpZBggJI0,21184
|
|
39
39
|
tinybird/tb/modules/deployment_common.py,sha256=fLPWNNs8ZwSkWHJjLeqQUqTbKqnnhW1y8yKffBdDCo0,31948
|
|
40
40
|
tinybird/tb/modules/deprecations.py,sha256=XXekrzPO9v12F1ToQDUGzLYJJ2wrEUlGKOkLCSdfHiM,4935
|
|
41
41
|
tinybird/tb/modules/endpoint.py,sha256=yRTh6rQFJ98LA0hSC8rPD3EcDaJj41gk9oCsgcZPu_c,12112
|
|
42
42
|
tinybird/tb/modules/exceptions.py,sha256=_1BHy0OixEkeF0fOCu1_1Pj8pJS4BNfUyucqCViJGTw,5958
|
|
43
|
-
tinybird/tb/modules/feedback_manager.py,sha256=
|
|
43
|
+
tinybird/tb/modules/feedback_manager.py,sha256=_tlMzju93UaOX7rk_EBpl95IdTqIC53DMBYZuKFph0A,77673
|
|
44
44
|
tinybird/tb/modules/fmt.py,sha256=ejQC1-2mK42saA2R9DA-CENYgu06SUoiCoX4bCtRXT8,3734
|
|
45
|
-
tinybird/tb/modules/info.py,sha256=
|
|
45
|
+
tinybird/tb/modules/info.py,sha256=oO2DpPT-hpQ2Gf_TBAJHQ_xIfc1WCBpiGWhiIeM_K4w,13042
|
|
46
46
|
tinybird/tb/modules/infra.py,sha256=J9Noe9aZo5Y9ZKAhqh9jnv8azfivXLLHQ2a9TeMWN9s,32673
|
|
47
47
|
tinybird/tb/modules/job.py,sha256=gjUceRkepgseQe1Q23G8B25R06r2UfdEIIXlIaqBA5E,3079
|
|
48
48
|
tinybird/tb/modules/job_common.py,sha256=3rdRH9F9kCRL_dBa5fghB27xgHqnO3oulBeIb1AcSbE,687
|
|
@@ -58,7 +58,7 @@ tinybird/tb/modules/logs.py,sha256=uSQi_A6QIFOjAoj1h8XOI_51AQ3uk5UEg2TjYbZxDJA,2
|
|
|
58
58
|
tinybird/tb/modules/materialization.py,sha256=SaomNeaAzLWtcnsZdetYBxEq0ihY1cRzh23n3Z1P_c4,5643
|
|
59
59
|
tinybird/tb/modules/open.py,sha256=ddABA4guIrBhrCKXYpkH4cfiAcFOG2B1eVePk5BEXto,1799
|
|
60
60
|
tinybird/tb/modules/pipe.py,sha256=xPKtezhnWZ6k_g82r4XpgKslofhuIxb_PvynH4gdUzI,2393
|
|
61
|
-
tinybird/tb/modules/preview.py,sha256=
|
|
61
|
+
tinybird/tb/modules/preview.py,sha256=712lVUgFE2a1qCmBGVrU_JyiaRyo5-S0dypcBxmKlnM,4659
|
|
62
62
|
tinybird/tb/modules/project.py,sha256=Js8f_VjzEI3cITaw5a_X2wgZ6kiO-VQq8LSx688ScDA,8697
|
|
63
63
|
tinybird/tb/modules/project_commands.py,sha256=t7h5JJWNA5eGth_Nrj0fak5B9eyVPGeDluF-41OBkhw,1877
|
|
64
64
|
tinybird/tb/modules/py_project.py,sha256=DE3N-GPhcvVulELMINZQmw0zG6xZ1YeVkyJt1vn2E4I,7799
|
|
@@ -76,7 +76,7 @@ tinybird/tb/modules/ts_project.py,sha256=XeDN-3gthT7GZYIFn0l-y5RJaofB_2T-I8gq6dL
|
|
|
76
76
|
tinybird/tb/modules/watch.py,sha256=sXv-Un5d9S_p4P_I4aQe_StRJCXR_NloAxFFRPDs6Hw,8734
|
|
77
77
|
tinybird/tb/modules/workspace.py,sha256=7shWyyZD9IT32EeP0aEgsZGk0U1ZdT6hWnVfctRoIJk,8939
|
|
78
78
|
tinybird/tb/modules/workspace_members.py,sha256=8oQLTczEh9cIdD3iF-N3SJuqLtKdF-_g-YzeVaqKGP0,9486
|
|
79
|
-
tinybird/tb/modules/datafile/build.py,sha256=
|
|
79
|
+
tinybird/tb/modules/datafile/build.py,sha256=Xd9umDi1BvJbirl0N_1rZ-DijvoU7DgZiURiE7DpA9o,52005
|
|
80
80
|
tinybird/tb/modules/datafile/build_common.py,sha256=2yNdxe49IMA9wNvl25NemY2Iaz8L66snjOdT64dm1is,4511
|
|
81
81
|
tinybird/tb/modules/datafile/build_datasource.py,sha256=keAx2lchYoxEDXQfTBtAe94Oyr4zfw3kWMumBFNtYso,16914
|
|
82
82
|
tinybird/tb/modules/datafile/build_pipe.py,sha256=VYrQULliQ2fs4rLvRvwJwI0L45zrWa15CP_09CLP03Y,11225
|
|
@@ -97,8 +97,8 @@ tinybird/tb_cli_modules/config.py,sha256=0kFDmsDcjKon32rgFGMHHKSbv4j5dOrXtVOlyuA
|
|
|
97
97
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
98
98
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
99
99
|
tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
|
|
100
|
-
tinybird-4.5.
|
|
101
|
-
tinybird-4.5.
|
|
102
|
-
tinybird-4.5.
|
|
103
|
-
tinybird-4.5.
|
|
104
|
-
tinybird-4.5.
|
|
100
|
+
tinybird-4.5.12.dev0.dist-info/METADATA,sha256=XbnvMWb-DpNVAEBHXrnCp54kdLoqC17QPIM26bgyUX4,12487
|
|
101
|
+
tinybird-4.5.12.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
|
102
|
+
tinybird-4.5.12.dev0.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
103
|
+
tinybird-4.5.12.dev0.dist-info/top_level.txt,sha256=ZIQJTPCzMqnfDzM_hEGZrJqDSEcKnIK_49T86DGWpyQ,78
|
|
104
|
+
tinybird-4.5.12.dev0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|