tinybird-cli 6.5.0__tar.gz → 6.5.3.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/PKG-INFO +8 -2
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/__cli__.py +2 -2
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/client.py +0 -6
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/datafile_common.py +24 -4
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/sql.py +8 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/sql_template.py +2 -2
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/branch.py +14 -12
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/common.py +3 -3
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/connection.py +80 -9
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird_cli.egg-info/PKG-INFO +8 -2
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird_cli.egg-info/SOURCES.txt +0 -1
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird_cli.egg-info/requires.txt +1 -1
- tinybird_cli-6.5.0/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -65
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/setup.cfg +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/ch_utils/constants.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/ch_utils/engine.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/check_pypi.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/config.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/context.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/datatypes.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/feedback_manager.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/git_settings.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/sql_template_fmt.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/sql_toolset.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/syncasync.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird/tornado_template.py +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird_cli-6.5.0 → tinybird_cli-6.5.3.dev0}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 6.5.
|
|
3
|
+
Version: 6.5.3.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -22,7 +22,7 @@ Requires-Dist: requests<3,>=2.28.1
|
|
|
22
22
|
Requires-Dist: shandy-sqlfmt==0.11.1
|
|
23
23
|
Requires-Dist: shandy-sqlfmt[jinjafmt]==0.11.1
|
|
24
24
|
Requires-Dist: toposort==1.10
|
|
25
|
-
Requires-Dist: tornado~=6.
|
|
25
|
+
Requires-Dist: tornado~=6.5.5
|
|
26
26
|
Requires-Dist: urllib3<2,>=1.26.14
|
|
27
27
|
Requires-Dist: wheel
|
|
28
28
|
Requires-Dist: packaging<24,>=23.1
|
|
@@ -43,6 +43,12 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
43
43
|
Changelog
|
|
44
44
|
----------
|
|
45
45
|
|
|
46
|
+
6.5.1
|
|
47
|
+
***********
|
|
48
|
+
|
|
49
|
+
- `Added` `tb connection create kafka` accepts four new flags — ``--oauthbearer-method``, ``--oauthbearer-aws-region``, ``--oauthbearer-aws-role-arn``, ``--oauthbearer-aws-external-id`` — for creating Amazon MSK connections with SASL ``OAUTHBEARER`` (IAM) authentication. Prompts interactively for region, role ARN, and external ID when the flags are omitted.
|
|
50
|
+
- `Changed` `tb branch create` now shows a deprecation warning (instead of failing) when `--ignore-datasource` is provided, and continues by ignoring that flag.
|
|
51
|
+
|
|
46
52
|
6.5.0
|
|
47
53
|
***********
|
|
48
54
|
|
|
@@ -770,14 +770,11 @@ class TinyB:
|
|
|
770
770
|
branch_name: str,
|
|
771
771
|
last_partition: Optional[bool],
|
|
772
772
|
all: Optional[bool],
|
|
773
|
-
ignore_datasources: Optional[List[str]],
|
|
774
773
|
):
|
|
775
774
|
params = {
|
|
776
775
|
"name": branch_name,
|
|
777
776
|
"data": LAST_PARTITION if last_partition else (ALL_PARTITIONS if all else ""),
|
|
778
777
|
}
|
|
779
|
-
if ignore_datasources:
|
|
780
|
-
params["ignore_datasources"] = ",".join(ignore_datasources)
|
|
781
778
|
return await self._req(f"/v0/environments?{urlencode(params)}", method="POST", data=b"")
|
|
782
779
|
|
|
783
780
|
async def branch_workspace_data(
|
|
@@ -785,7 +782,6 @@ class TinyB:
|
|
|
785
782
|
workspace_id: str,
|
|
786
783
|
last_partition: bool,
|
|
787
784
|
all: bool,
|
|
788
|
-
ignore_datasources: Optional[List[str]] = None,
|
|
789
785
|
):
|
|
790
786
|
params = {}
|
|
791
787
|
if last_partition:
|
|
@@ -793,8 +789,6 @@ class TinyB:
|
|
|
793
789
|
|
|
794
790
|
if all:
|
|
795
791
|
params["mode"] = ALL_PARTITIONS
|
|
796
|
-
if ignore_datasources:
|
|
797
|
-
params["ignore_datasources"] = ",".join(ignore_datasources)
|
|
798
792
|
url = f"/v0/environments/{workspace_id}/data?{urlencode(params)}"
|
|
799
793
|
return await self._req(url, method="POST", data=b"")
|
|
800
794
|
|
|
@@ -1373,6 +1373,9 @@ async def process_file(
|
|
|
1373
1373
|
if not skip_connectors:
|
|
1374
1374
|
try:
|
|
1375
1375
|
is_oauthbearer = params.get("kafka_sasl_mechanism") == "OAUTHBEARER"
|
|
1376
|
+
# PLAINTEXT brokers have no SASL at all, so a key/secret pair is
|
|
1377
|
+
# neither produced by the wizard nor expected on the wire.
|
|
1378
|
+
is_plaintext = params.get("kafka_security_protocol") == "PLAINTEXT"
|
|
1376
1379
|
connector_params = {
|
|
1377
1380
|
"kafka_bootstrap_servers": params.get("kafka_bootstrap_servers", None),
|
|
1378
1381
|
"kafka_key": params.get("kafka_key", None),
|
|
@@ -1382,6 +1385,10 @@ async def process_file(
|
|
|
1382
1385
|
"kafka_schema_registry_url": params.get("kafka_schema_registry_url", None),
|
|
1383
1386
|
"kafka_ssl_ca_pem": get_ca_pem_content(params.get("kafka_ssl_ca_pem", None), filename),
|
|
1384
1387
|
"kafka_sasl_mechanism": params.get("kafka_sasl_mechanism", None),
|
|
1388
|
+
# Explicit default — None would be dropped by the client's
|
|
1389
|
+
# `if value is not None` filter, leaving the server to pick
|
|
1390
|
+
# its own protocol instead of the SASL_SSL we want.
|
|
1391
|
+
"kafka_security_protocol": params.get("kafka_security_protocol", "SASL_SSL"),
|
|
1385
1392
|
"kafka_sasl_oauthbearer_method": params.get("kafka_sasl_oauthbearer_method", None),
|
|
1386
1393
|
"kafka_sasl_oauthbearer_aws_region": params.get("kafka_sasl_oauthbearer_aws_region", None),
|
|
1387
1394
|
"kafka_sasl_oauthbearer_aws_role_arn": params.get("kafka_sasl_oauthbearer_aws_role_arn", None),
|
|
@@ -1402,6 +1409,11 @@ async def process_file(
|
|
|
1402
1409
|
connector_params["kafka_sasl_oauthbearer_aws_region"],
|
|
1403
1410
|
connector_params["kafka_sasl_oauthbearer_aws_role_arn"],
|
|
1404
1411
|
]
|
|
1412
|
+
elif is_plaintext:
|
|
1413
|
+
# No SASL, no credentials — only the broker address matters.
|
|
1414
|
+
required_params = [
|
|
1415
|
+
connector_params["kafka_bootstrap_servers"],
|
|
1416
|
+
]
|
|
1405
1417
|
else:
|
|
1406
1418
|
required_params = [
|
|
1407
1419
|
connector_params["kafka_bootstrap_servers"],
|
|
@@ -3134,7 +3146,7 @@ async def new_pipe(
|
|
|
3134
3146
|
scopes.append(sc)
|
|
3135
3147
|
try:
|
|
3136
3148
|
r = await tb_client.alter_tokens(token_name, scopes)
|
|
3137
|
-
token = r["token"] # type: ignore
|
|
3149
|
+
token = r["token"] # type: ignore[index, unused-ignore]
|
|
3138
3150
|
except Exception as e:
|
|
3139
3151
|
raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
|
|
3140
3152
|
else:
|
|
@@ -3144,7 +3156,7 @@ async def new_pipe(
|
|
|
3144
3156
|
r = await tb_client.create_token(
|
|
3145
3157
|
token_name, [f"PIPES:{tk['permissions']}:{p['name']}"], "P", p["name"]
|
|
3146
3158
|
)
|
|
3147
|
-
token = r["token"] # type: ignore
|
|
3159
|
+
token = r["token"] # type: ignore[index, unused-ignore]
|
|
3148
3160
|
except Exception as e:
|
|
3149
3161
|
raise click.ClickException(FeedbackManager.error_creating_pipe(error=e))
|
|
3150
3162
|
|
|
@@ -5282,8 +5294,16 @@ async def _gather_with_concurrency(n, *tasks):
|
|
|
5282
5294
|
semaphore = Semaphore(n)
|
|
5283
5295
|
|
|
5284
5296
|
async def sem_task(task):
|
|
5285
|
-
|
|
5286
|
-
|
|
5297
|
+
try:
|
|
5298
|
+
async with semaphore:
|
|
5299
|
+
return await task
|
|
5300
|
+
finally:
|
|
5301
|
+
# If we never reached `await task` (e.g. cancelled while waiting
|
|
5302
|
+
# on the semaphore after a sibling raised), close the coroutine
|
|
5303
|
+
# so it doesn't surface as a "coroutine was never awaited"
|
|
5304
|
+
# RuntimeWarning at GC time. close() is a no-op on an already
|
|
5305
|
+
# exhausted coroutine, so the happy path is unaffected.
|
|
5306
|
+
task.close()
|
|
5287
5307
|
|
|
5288
5308
|
wrapped = [ensure_future(sem_task(t)) for t in tasks]
|
|
5289
5309
|
try:
|
|
@@ -398,6 +398,14 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
|
|
|
398
398
|
>>> parse_table_structure('c Nullable(Float32) DEFAULT NULL')
|
|
399
399
|
[{'name': 'c', 'type': 'Float32', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': True, 'normalized_name': 'c'}]
|
|
400
400
|
|
|
401
|
+
>>> parse_table_structure('c LowCardinality(Nullable(String))')
|
|
402
|
+
[{'name': 'c', 'type': 'LowCardinality(Nullable(String))', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
|
|
403
|
+
|
|
404
|
+
>>> parse_table_structure('c Nullable(LowCardinality(Nullable(String)))')
|
|
405
|
+
Traceback (most recent call last):
|
|
406
|
+
...
|
|
407
|
+
ValueError: Nested type LowCardinality(Nullable(String)) cannot be inside Nullable type
|
|
408
|
+
|
|
401
409
|
>>> parse_table_structure("c String DEFAULT 'bla'")
|
|
402
410
|
[{'name': 'c', 'type': 'String', 'codec': None, 'default_value': "DEFAULT 'bla'", 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
|
|
403
411
|
|
|
@@ -7,7 +7,7 @@ from collections import deque
|
|
|
7
7
|
from datetime import datetime
|
|
8
8
|
from functools import lru_cache
|
|
9
9
|
from json import loads
|
|
10
|
-
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
|
10
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
|
11
11
|
|
|
12
12
|
from tornado import escape
|
|
13
13
|
from tornado.util import ObjectDict, exec_in, unicode_type
|
|
@@ -1499,7 +1499,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
|
|
|
1499
1499
|
)
|
|
1500
1500
|
|
|
1501
1501
|
exec_in(self.compiled, namespace)
|
|
1502
|
-
execute = namespace["_tt_execute"]
|
|
1502
|
+
execute = cast(Callable[[], bytes], namespace["_tt_execute"])
|
|
1503
1503
|
# Clear the traceback module's cache of source data now that
|
|
1504
1504
|
# we've generated a new template (mainly for this module's
|
|
1505
1505
|
# unittests, where different tests reuse the same name).
|
|
@@ -389,11 +389,21 @@ async def branch_current() -> None:
|
|
|
389
389
|
)
|
|
390
390
|
@coro
|
|
391
391
|
async def create_branch(
|
|
392
|
-
branch_name: Optional[str],
|
|
392
|
+
branch_name: Optional[str],
|
|
393
|
+
last_partition: bool,
|
|
394
|
+
all: bool,
|
|
395
|
+
ignore_datasources: Tuple[str, ...],
|
|
396
|
+
wait: bool,
|
|
393
397
|
) -> None:
|
|
394
398
|
if last_partition and all:
|
|
395
399
|
raise CLIException(FeedbackManager.error_exception(error="Use --last-partition or --all but not both"))
|
|
396
|
-
|
|
400
|
+
if ignore_datasources:
|
|
401
|
+
click.echo(
|
|
402
|
+
FeedbackManager.warning_deprecated(
|
|
403
|
+
warning="--ignore-datasource is no longer supported for `tb branch create` and will be ignored."
|
|
404
|
+
)
|
|
405
|
+
)
|
|
406
|
+
await create_workspace_branch(branch_name, last_partition, all, wait)
|
|
397
407
|
|
|
398
408
|
|
|
399
409
|
@branch.command(name="rm", short_help="Removes a Branch from the Workspace. It can't be recovered.")
|
|
@@ -468,14 +478,6 @@ async def delete_branch(branch_name_or_id: str, yes: bool) -> None:
|
|
|
468
478
|
help="Attach all data from 'main' to the new Branch. Use only if you actually need all the data in the Branch",
|
|
469
479
|
hidden=True,
|
|
470
480
|
)
|
|
471
|
-
@click.option(
|
|
472
|
-
"-i",
|
|
473
|
-
"--ignore-datasource",
|
|
474
|
-
"ignore_datasources",
|
|
475
|
-
type=str,
|
|
476
|
-
multiple=True,
|
|
477
|
-
help="Ignore specified data source partitions",
|
|
478
|
-
)
|
|
479
481
|
@click.option(
|
|
480
482
|
"--wait",
|
|
481
483
|
is_flag=True,
|
|
@@ -483,7 +485,7 @@ async def delete_branch(branch_name_or_id: str, yes: bool) -> None:
|
|
|
483
485
|
help="Wait for data branch jobs to finish, showing a progress bar. Disabled by default.",
|
|
484
486
|
)
|
|
485
487
|
@coro
|
|
486
|
-
async def data_branch(last_partition: bool, all: bool,
|
|
488
|
+
async def data_branch(last_partition: bool, all: bool, wait: bool) -> None:
|
|
487
489
|
if last_partition and all:
|
|
488
490
|
raise CLIException(FeedbackManager.error_exception(error="Use --last-partition or --all but not both"))
|
|
489
491
|
|
|
@@ -500,7 +502,7 @@ async def data_branch(last_partition: bool, all: bool, ignore_datasources: List[
|
|
|
500
502
|
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
501
503
|
|
|
502
504
|
try:
|
|
503
|
-
response = await client.branch_workspace_data(config["id"], last_partition, all
|
|
505
|
+
response = await client.branch_workspace_data(config["id"], last_partition, all)
|
|
504
506
|
|
|
505
507
|
is_job: bool = "job" in response
|
|
506
508
|
is_summary: bool = "partitions" in response
|
|
@@ -112,7 +112,9 @@ def echo_safe_humanfriendly_tables_format_smart_table(data: Iterable[Any], colum
|
|
|
112
112
|
try:
|
|
113
113
|
click.echo(humanfriendly.tables.format_smart_table(data, column_names=column_names))
|
|
114
114
|
except ValueError as exc:
|
|
115
|
-
|
|
115
|
+
# Python 3.11 wording: "max() arg is an empty sequence"
|
|
116
|
+
# Python 3.12 wording: "max() iterable argument is empty"
|
|
117
|
+
if str(exc) in ("max() arg is an empty sequence", "max() iterable argument is empty"):
|
|
116
118
|
click.echo("------------")
|
|
117
119
|
click.echo("Empty")
|
|
118
120
|
click.echo("------------")
|
|
@@ -677,7 +679,6 @@ async def create_workspace_branch(
|
|
|
677
679
|
branch_name: Optional[str],
|
|
678
680
|
last_partition: bool,
|
|
679
681
|
all: bool,
|
|
680
|
-
ignore_datasources: Optional[List[str]],
|
|
681
682
|
wait: Optional[bool],
|
|
682
683
|
) -> None:
|
|
683
684
|
"""
|
|
@@ -701,7 +702,6 @@ async def create_workspace_branch(
|
|
|
701
702
|
branch_name,
|
|
702
703
|
last_partition,
|
|
703
704
|
all,
|
|
704
|
-
ignore_datasources,
|
|
705
705
|
)
|
|
706
706
|
assert isinstance(response, dict)
|
|
707
707
|
|
|
@@ -35,6 +35,18 @@ from tinybird.tb_cli_modules.common import (
|
|
|
35
35
|
)
|
|
36
36
|
from tinybird.tb_cli_modules.exceptions import CLIConnectionException
|
|
37
37
|
|
|
38
|
+
|
|
39
|
+
def _upper_or_none(_ctx: click.Context, _param: click.Parameter, value: Optional[str]) -> Optional[str]:
|
|
40
|
+
"""Click callback that uppercases the option value if present.
|
|
41
|
+
|
|
42
|
+
The async client matches Kafka SASL mechanisms case-sensitively (see
|
|
43
|
+
``tinybird/client.py``), so we normalize at the flag layer to avoid the
|
|
44
|
+
mismatch where the CLI treats ``--sasl-mechanism oauthbearer`` as IAM
|
|
45
|
+
but the client falls through and sends PLAIN credentials.
|
|
46
|
+
"""
|
|
47
|
+
return value.upper() if value else value
|
|
48
|
+
|
|
49
|
+
|
|
38
50
|
DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
39
51
|
DataConnectorType.KAFKA: [
|
|
40
52
|
"kafka_bootstrap_servers",
|
|
@@ -46,6 +58,10 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
|
46
58
|
"kafka_sasl_mechanism",
|
|
47
59
|
"kafka_schema_registry_url",
|
|
48
60
|
"kafka_ssl_ca_pem",
|
|
61
|
+
"kafka_sasl_oauthbearer_method",
|
|
62
|
+
"kafka_sasl_oauthbearer_aws_region",
|
|
63
|
+
"kafka_sasl_oauthbearer_aws_role_arn",
|
|
64
|
+
"kafka_sasl_oauthbearer_aws_external_id",
|
|
49
65
|
],
|
|
50
66
|
DataConnectorType.GCLOUD_SCHEDULER: ["gcscheduler_region"],
|
|
51
67
|
DataConnectorType.GCLOUD_STORAGE: [
|
|
@@ -79,7 +95,7 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
|
79
95
|
}
|
|
80
96
|
|
|
81
97
|
SENSITIVE_CONNECTOR_SETTINGS = {
|
|
82
|
-
DataConnectorType.KAFKA: ["kafka_sasl_plain_password"],
|
|
98
|
+
DataConnectorType.KAFKA: ["kafka_sasl_plain_password", "kafka_sasl_oauthbearer_aws_role_arn"],
|
|
83
99
|
DataConnectorType.GCLOUD_SCHEDULER: [
|
|
84
100
|
"gcscheduler_target_url",
|
|
85
101
|
"gcscheduler_job_name",
|
|
@@ -120,14 +136,33 @@ def connection_create(ctx: Context) -> None:
|
|
|
120
136
|
@click.option(
|
|
121
137
|
"--sasl-mechanism",
|
|
122
138
|
default="PLAIN",
|
|
139
|
+
callback=_upper_or_none,
|
|
123
140
|
help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
|
|
124
141
|
)
|
|
125
142
|
@click.option(
|
|
126
143
|
"--security-protocol",
|
|
127
144
|
default="SASL_SSL",
|
|
145
|
+
callback=_upper_or_none,
|
|
128
146
|
help="Security protocol for connection-based protocols. Defaults to 'SASL_SSL'",
|
|
129
147
|
)
|
|
130
148
|
@click.option("--ssl-ca-pem", default=None, help="Path or content of the CA Certificate file in PEM format")
|
|
149
|
+
@click.option(
|
|
150
|
+
"--oauthbearer-method",
|
|
151
|
+
default=None,
|
|
152
|
+
callback=_upper_or_none,
|
|
153
|
+
help="OAUTHBEARER token provider. Use 'AWS' for Amazon MSK IAM authentication.",
|
|
154
|
+
)
|
|
155
|
+
@click.option("--oauthbearer-aws-region", default=None, help="AWS region of the MSK cluster (OAUTHBEARER+AWS only)")
|
|
156
|
+
@click.option(
|
|
157
|
+
"--oauthbearer-aws-role-arn",
|
|
158
|
+
default=None,
|
|
159
|
+
help="ARN of the IAM role to assume for the MSK cluster (OAUTHBEARER+AWS only)",
|
|
160
|
+
)
|
|
161
|
+
@click.option(
|
|
162
|
+
"--oauthbearer-aws-external-id",
|
|
163
|
+
default=None,
|
|
164
|
+
help="Optional external_id for the AWS assume-role call. If omitted the server derives it from the workspace.",
|
|
165
|
+
)
|
|
131
166
|
@click.pass_context
|
|
132
167
|
@coro
|
|
133
168
|
async def connection_create_kafka(
|
|
@@ -141,6 +176,10 @@ async def connection_create_kafka(
|
|
|
141
176
|
sasl_mechanism: Optional[str],
|
|
142
177
|
security_protocol: Optional[str],
|
|
143
178
|
ssl_ca_pem: Optional[str],
|
|
179
|
+
oauthbearer_method: Optional[str],
|
|
180
|
+
oauthbearer_aws_region: Optional[str],
|
|
181
|
+
oauthbearer_aws_role_arn: Optional[str],
|
|
182
|
+
oauthbearer_aws_external_id: Optional[str],
|
|
144
183
|
) -> None:
|
|
145
184
|
"""
|
|
146
185
|
Add a Kafka connection
|
|
@@ -149,21 +188,39 @@ async def connection_create_kafka(
|
|
|
149
188
|
$ tb connection create kafka --bootstrap-servers google.com:80 --key a --secret b --connection-name c
|
|
150
189
|
"""
|
|
151
190
|
|
|
191
|
+
# `--sasl-mechanism` arrives uppercased via the _upper_or_none callback.
|
|
192
|
+
is_oauthbearer = sasl_mechanism == "OAUTHBEARER"
|
|
193
|
+
|
|
152
194
|
bootstrap_servers and validate_kafka_bootstrap_servers(bootstrap_servers)
|
|
153
|
-
|
|
154
|
-
|
|
195
|
+
if not is_oauthbearer:
|
|
196
|
+
key and validate_kafka_key(key)
|
|
197
|
+
secret and validate_kafka_secret(secret)
|
|
155
198
|
schema_registry_url and validate_kafka_schema_registry_url(schema_registry_url)
|
|
156
199
|
auto_offset_reset and validate_kafka_auto_offset_reset(auto_offset_reset)
|
|
157
200
|
|
|
158
201
|
if not bootstrap_servers:
|
|
159
202
|
bootstrap_servers = click.prompt("Kafka Bootstrap Server")
|
|
160
203
|
validate_kafka_bootstrap_servers(bootstrap_servers)
|
|
161
|
-
if
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
204
|
+
if is_oauthbearer:
|
|
205
|
+
if not oauthbearer_method:
|
|
206
|
+
oauthbearer_method = "AWS"
|
|
207
|
+
if not oauthbearer_aws_region:
|
|
208
|
+
oauthbearer_aws_region = click.prompt("AWS region (e.g. us-east-1)")
|
|
209
|
+
if not oauthbearer_aws_role_arn:
|
|
210
|
+
oauthbearer_aws_role_arn = click.prompt("AWS IAM role ARN")
|
|
211
|
+
if not oauthbearer_aws_external_id:
|
|
212
|
+
# Pre-shared external_id (set by the cluster owner in the role's trust
|
|
213
|
+
# policy). Empty answer = let the server derive one from the workspace.
|
|
214
|
+
provided = click.prompt("External ID (optional, leave blank to let the server derive one)", default="")
|
|
215
|
+
if provided and provided.strip():
|
|
216
|
+
oauthbearer_aws_external_id = provided.strip()
|
|
217
|
+
else:
|
|
218
|
+
if key is None:
|
|
219
|
+
key = click.prompt("Key")
|
|
220
|
+
validate_kafka_key(key)
|
|
221
|
+
if secret is None:
|
|
222
|
+
secret = click.prompt("Secret", hide_input=True)
|
|
223
|
+
validate_kafka_secret(secret)
|
|
167
224
|
if not connection_name:
|
|
168
225
|
connection_name = click.prompt(
|
|
169
226
|
f"Connection name (optional, current: {bootstrap_servers})", default=bootstrap_servers
|
|
@@ -172,6 +229,19 @@ async def connection_create_kafka(
|
|
|
172
229
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
173
230
|
client: TinyB = obj["client"]
|
|
174
231
|
|
|
232
|
+
# Only forward the OAUTHBEARER kwargs when the user actually picked OAUTHBEARER.
|
|
233
|
+
# The receiving client accepts them as kwargs either way, but keeping the call
|
|
234
|
+
# symmetric with the auth mechanism makes the intent obvious to readers
|
|
235
|
+
oauthbearer_kwargs = (
|
|
236
|
+
{
|
|
237
|
+
"kafka_sasl_oauthbearer_method": oauthbearer_method,
|
|
238
|
+
"kafka_sasl_oauthbearer_aws_region": oauthbearer_aws_region,
|
|
239
|
+
"kafka_sasl_oauthbearer_aws_role_arn": oauthbearer_aws_role_arn,
|
|
240
|
+
"kafka_sasl_oauthbearer_aws_external_id": oauthbearer_aws_external_id,
|
|
241
|
+
}
|
|
242
|
+
if is_oauthbearer
|
|
243
|
+
else {}
|
|
244
|
+
)
|
|
175
245
|
result = await client.connection_create_kafka(
|
|
176
246
|
bootstrap_servers,
|
|
177
247
|
key,
|
|
@@ -182,6 +252,7 @@ async def connection_create_kafka(
|
|
|
182
252
|
sasl_mechanism,
|
|
183
253
|
security_protocol,
|
|
184
254
|
get_ca_pem_content(ssl_ca_pem),
|
|
255
|
+
**oauthbearer_kwargs,
|
|
185
256
|
)
|
|
186
257
|
|
|
187
258
|
id = result["id"]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version: 6.5.
|
|
3
|
+
Version: 6.5.3.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
@@ -22,7 +22,7 @@ Requires-Dist: requests<3,>=2.28.1
|
|
|
22
22
|
Requires-Dist: shandy-sqlfmt==0.11.1
|
|
23
23
|
Requires-Dist: shandy-sqlfmt[jinjafmt]==0.11.1
|
|
24
24
|
Requires-Dist: toposort==1.10
|
|
25
|
-
Requires-Dist: tornado~=6.
|
|
25
|
+
Requires-Dist: tornado~=6.5.5
|
|
26
26
|
Requires-Dist: urllib3<2,>=1.26.14
|
|
27
27
|
Requires-Dist: wheel
|
|
28
28
|
Requires-Dist: packaging<24,>=23.1
|
|
@@ -43,6 +43,12 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
43
43
|
Changelog
|
|
44
44
|
----------
|
|
45
45
|
|
|
46
|
+
6.5.1
|
|
47
|
+
***********
|
|
48
|
+
|
|
49
|
+
- `Added` `tb connection create kafka` accepts four new flags — ``--oauthbearer-method``, ``--oauthbearer-aws-region``, ``--oauthbearer-aws-role-arn``, ``--oauthbearer-aws-external-id`` — for creating Amazon MSK connections with SASL ``OAUTHBEARER`` (IAM) authentication. Prompts interactively for region, role ARN, and external ID when the flags are omitted.
|
|
50
|
+
- `Changed` `tb branch create` now shows a deprecation warning (instead of failing) when `--ignore-datasource` is provided, and continues by ignoring that flag.
|
|
51
|
+
|
|
46
52
|
6.5.0
|
|
47
53
|
***********
|
|
48
54
|
|
|
@@ -36,7 +36,6 @@ tinybird/tb_cli_modules/token.py
|
|
|
36
36
|
tinybird/tb_cli_modules/workspace.py
|
|
37
37
|
tinybird/tb_cli_modules/workspace_members.py
|
|
38
38
|
tinybird/tb_cli_modules/tinyunit/tinyunit.py
|
|
39
|
-
tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py
|
|
40
39
|
tinybird_cli.egg-info/PKG-INFO
|
|
41
40
|
tinybird_cli.egg-info/SOURCES.txt
|
|
42
41
|
tinybird_cli.egg-info/dependency_links.txt
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import json
|
|
2
|
-
from collections import namedtuple
|
|
3
|
-
from json import JSONEncoder
|
|
4
|
-
from typing import Optional
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class MyJSONEncoder(JSONEncoder):
|
|
8
|
-
# def default(self, in_obj):
|
|
9
|
-
# loaded_data = [
|
|
10
|
-
# DataUnitTest(
|
|
11
|
-
# getattr(unitDataTest, 'name'),
|
|
12
|
-
# getattr(unitDataTest, 'description'),
|
|
13
|
-
# getattr(unitDataTest, 'enabled'),
|
|
14
|
-
# getattr(unitDataTest, 'endpoint'),
|
|
15
|
-
# getattr(unitDataTest, 'result'),
|
|
16
|
-
# getattr(unitDataTest, 'time'),
|
|
17
|
-
# getattr(unitDataTest, 'sql'))
|
|
18
|
-
# for unitDataTest in in_obj]
|
|
19
|
-
# return loaded_data
|
|
20
|
-
def default(self, obj):
|
|
21
|
-
return obj.to_json()
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
class DataUnitTest:
|
|
25
|
-
def __init__(
|
|
26
|
-
self,
|
|
27
|
-
name: str,
|
|
28
|
-
description: str,
|
|
29
|
-
enabled: bool,
|
|
30
|
-
endpoint: Optional[str],
|
|
31
|
-
result: Optional[str],
|
|
32
|
-
time: int,
|
|
33
|
-
sql: str,
|
|
34
|
-
):
|
|
35
|
-
self.name = name
|
|
36
|
-
self.description = description
|
|
37
|
-
self.enabled = enabled
|
|
38
|
-
self.endpoint = endpoint
|
|
39
|
-
self.result = result
|
|
40
|
-
self.time = time
|
|
41
|
-
self.sql = sql
|
|
42
|
-
|
|
43
|
-
def __iter__(self):
|
|
44
|
-
yield from {
|
|
45
|
-
"name": self.name,
|
|
46
|
-
"description": self.description,
|
|
47
|
-
"enabled": self.enabled,
|
|
48
|
-
"endpoint": self.endpoint,
|
|
49
|
-
"sql": self.sql,
|
|
50
|
-
"result": self.result,
|
|
51
|
-
"time": self.time,
|
|
52
|
-
}.items()
|
|
53
|
-
|
|
54
|
-
def __str__(self):
|
|
55
|
-
return json.dumps(dict(self), ensure_ascii=False)
|
|
56
|
-
|
|
57
|
-
def __repr__(self):
|
|
58
|
-
return self.__str__()
|
|
59
|
-
|
|
60
|
-
def to_json(self):
|
|
61
|
-
return self.__str__()
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def customDataUnitTestDecoder(dataUnitTestDict):
|
|
65
|
-
return namedtuple("X", dataUnitTestDict.keys())(*dataUnitTestDict.values())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|