tinybird-cli 5.2.2.dev1__tar.gz → 5.2.2.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/PKG-INFO +13 -1
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/__cli__.py +2 -2
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/ch_utils/engine.py +10 -1
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/client.py +6 -6
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/datafile.py +7 -3
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/feedback_manager.py +8 -2
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/common.py +192 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/connection.py +69 -158
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/PKG-INFO +13 -1
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/setup.cfg +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/check_pypi.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/config.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/connectors.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/context.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/datatypes.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/git_settings.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/sql.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/sql_template.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/sql_toolset.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/syncasync.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tornado_template.py +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/SOURCES.txt +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.2.2.
|
|
3
|
+
Version: 5.2.2.dev3
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,8 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
+
|
|
22
|
+
5.2.2.dev3
|
|
23
|
+
**********
|
|
24
|
+
|
|
25
|
+
- `Added` `tb connection create` now supports `dynamodb` as service type
|
|
26
|
+
|
|
27
|
+
5.2.2.dev2
|
|
28
|
+
**********
|
|
29
|
+
|
|
30
|
+
- `Added` error when trying to push a data source with `SETTINGS` instead of `ENGINE_SETTINGS`
|
|
31
|
+
|
|
21
32
|
5.2.2.dev1
|
|
22
33
|
**********
|
|
34
|
+
|
|
23
35
|
- `Added` support for buckets with gzip files when creating S3 Data Sources
|
|
24
36
|
|
|
25
37
|
5.2.1
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '5.2.2.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '5.2.2.dev3'
|
|
8
|
+
__revision__ = 'decfb20'
|
|
@@ -196,6 +196,11 @@ class TableDetails:
|
|
|
196
196
|
_ver = self.details.get("ver", None)
|
|
197
197
|
return _ver
|
|
198
198
|
|
|
199
|
+
@property
|
|
200
|
+
def is_deleted(self):
|
|
201
|
+
_is_deleted = self.details.get("is_deleted", None)
|
|
202
|
+
return _is_deleted
|
|
203
|
+
|
|
199
204
|
@property
|
|
200
205
|
def columns(self):
|
|
201
206
|
_columns = self.details.get("columns", None)
|
|
@@ -256,6 +261,8 @@ class TableDetails:
|
|
|
256
261
|
d["key_columns"] = self.key_columns
|
|
257
262
|
if self.ver:
|
|
258
263
|
d["ver"] = self.ver
|
|
264
|
+
if self.is_deleted:
|
|
265
|
+
d["is_deleted"] = self.is_deleted
|
|
259
266
|
if self.sign:
|
|
260
267
|
d["sign"] = self.sign
|
|
261
268
|
if self.version:
|
|
@@ -424,7 +431,9 @@ ENABLED_ENGINES = [
|
|
|
424
431
|
engine_config("MergeTree", options=MERGETREE_OPTIONS),
|
|
425
432
|
# ReplacingMergeTree([ver])
|
|
426
433
|
engine_config(
|
|
427
|
-
"ReplacingMergeTree",
|
|
434
|
+
"ReplacingMergeTree",
|
|
435
|
+
[EngineParam(name="ver", is_valid=column_is_valid), EngineParam(name="is_deleted", is_valid=column_is_valid)],
|
|
436
|
+
REPLACINGMERGETREE_OPTIONS,
|
|
428
437
|
),
|
|
429
438
|
# SummingMergeTree([columns])
|
|
430
439
|
engine_config(
|
|
@@ -1011,14 +1011,14 @@ class TinyB(object):
|
|
|
1011
1011
|
response = await self._req(f"/v0/connectors/snowflake/warehouses?{urlencode(params)}", method="POST", data="")
|
|
1012
1012
|
return response["warehouses"]
|
|
1013
1013
|
|
|
1014
|
-
async def
|
|
1015
|
-
return await self._req("/v0/integrations/
|
|
1014
|
+
async def get_trust_policy(self, service: str) -> Dict[str, Any]:
|
|
1015
|
+
return await self._req(f"/v0/integrations/{service}/policies/trust-policy")
|
|
1016
1016
|
|
|
1017
|
-
async def
|
|
1018
|
-
return await self._req("/v0/integrations/
|
|
1017
|
+
async def get_access_write_policy(self, service: str) -> Dict[str, Any]:
|
|
1018
|
+
return await self._req(f"/v0/integrations/{service}/policies/write-access-policy")
|
|
1019
1019
|
|
|
1020
|
-
async def
|
|
1021
|
-
return await self._req("/v0/integrations/
|
|
1020
|
+
async def get_access_read_policy(self, service: str) -> Dict[str, Any]:
|
|
1021
|
+
return await self._req(f"/v0/integrations/{service}/policies/read-access-policy")
|
|
1022
1022
|
|
|
1023
1023
|
async def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
|
1024
1024
|
try:
|
|
@@ -1169,6 +1169,8 @@ def parse(
|
|
|
1169
1169
|
parser_state.command = cmd.lower()
|
|
1170
1170
|
parser_state.multiline_string = ""
|
|
1171
1171
|
else:
|
|
1172
|
+
if cmd.lower() == "settings":
|
|
1173
|
+
raise click.ClickException(FeedbackManager.error_settings_not_allowed())
|
|
1172
1174
|
if cmd.lower() in cmds:
|
|
1173
1175
|
cmds[cmd.lower()](*args, lineno=lineno, replace_includes=replace_includes)
|
|
1174
1176
|
else:
|
|
@@ -4114,9 +4116,11 @@ async def folder_push(
|
|
|
4114
4116
|
else:
|
|
4115
4117
|
click.echo(
|
|
4116
4118
|
FeedbackManager.warning_name_already_exists(
|
|
4117
|
-
name=
|
|
4118
|
-
|
|
4119
|
-
|
|
4119
|
+
name=(
|
|
4120
|
+
name
|
|
4121
|
+
if to_run[name]["version"] is None
|
|
4122
|
+
else f'{name}__v{to_run[name]["version"]}'
|
|
4123
|
+
)
|
|
4120
4124
|
)
|
|
4121
4125
|
)
|
|
4122
4126
|
else:
|
|
@@ -68,6 +68,9 @@ class FeedbackManager:
|
|
|
68
68
|
error_failed_to_format_files = error_message(
|
|
69
69
|
"{number} files need to be formatted, run the commands below to fix them."
|
|
70
70
|
)
|
|
71
|
+
error_settings_not_allowed = error_message(
|
|
72
|
+
"SETTINGS option is not allowed, use ENGINE_SETTINGS instead. See https://www.tinybird.co/docs/cli/datafiles#data-source for more information."
|
|
73
|
+
)
|
|
71
74
|
error_pipe_already_exists = error_message("{pipe} already exists")
|
|
72
75
|
error_pipe_node_same_name = error_message(
|
|
73
76
|
"Error in node '{name}'. The Pipe is already using that name. Nodes can't have the same exact name as the Pipe they belong to."
|
|
@@ -428,7 +431,7 @@ Ready? """
|
|
|
428
431
|
|
|
429
432
|
prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
|
|
430
433
|
prompt_s3_iamrole_connection_policy = prompt_message(
|
|
431
|
-
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace
|
|
434
|
+
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
|
|
432
435
|
)
|
|
433
436
|
prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
|
|
434
437
|
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""
|
|
@@ -473,6 +476,7 @@ Ready? """
|
|
|
473
476
|
)
|
|
474
477
|
info_creating_kafka_connection = info_message("** Creating new Kafka connection '{connection_name}'")
|
|
475
478
|
info_creating_s3_iamrole_connection = info_message("** Creating new S3 IAM Role connection '{connection_name}'")
|
|
479
|
+
info_creating_dynamodb_connection = info_message("** Creating new DynamoDB connection '{connection_name}'")
|
|
476
480
|
|
|
477
481
|
warning_remove_oldest_rollback = warning_message(
|
|
478
482
|
"** [WARNING] Will try to remove oldest rollback Release before promoting to live Release {semver}."
|
|
@@ -907,7 +911,9 @@ Ready? """
|
|
|
907
911
|
success_s3_iam_connection_created = success_message(
|
|
908
912
|
"** Info associated with this connection:\n** External ID: {external_id}\n** Role ARN: {role_arn}"
|
|
909
913
|
)
|
|
910
|
-
|
|
914
|
+
success_dynamodb_connection_created = success_message(
|
|
915
|
+
"** Info associated with this DynamoDB connection:\n** Region: {region}\n** Role ARN: {role_arn}"
|
|
916
|
+
)
|
|
911
917
|
success_delete_connection = success_message("** Connection {connection_id} removed successfully")
|
|
912
918
|
success_connection_using = success_message("** Using connection '{connection_name}'")
|
|
913
919
|
success_using_host = success_message("** Using host: {host} ({name})")
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import asyncio
|
|
9
9
|
import json
|
|
10
|
+
import os
|
|
10
11
|
import re
|
|
11
12
|
import socket
|
|
12
13
|
import sys
|
|
@@ -24,6 +25,7 @@ import click
|
|
|
24
25
|
import click.formatting
|
|
25
26
|
import humanfriendly
|
|
26
27
|
import humanfriendly.tables
|
|
28
|
+
import pyperclip
|
|
27
29
|
from click import Context
|
|
28
30
|
from click._termui_impl import ProgressBar
|
|
29
31
|
from humanfriendly.tables import format_pretty_table
|
|
@@ -1659,6 +1661,12 @@ class ConnectionReplacements:
|
|
|
1659
1661
|
"private_key_id": "gcs_private_key_id",
|
|
1660
1662
|
"connection_name": "name",
|
|
1661
1663
|
},
|
|
1664
|
+
"dynamodb": {
|
|
1665
|
+
"service": "service",
|
|
1666
|
+
"connection_name": "name",
|
|
1667
|
+
"role_arn": "dynamodb_iamrole_arn",
|
|
1668
|
+
"region": "dynamodb_iamrole_region",
|
|
1669
|
+
},
|
|
1662
1670
|
}
|
|
1663
1671
|
|
|
1664
1672
|
@staticmethod
|
|
@@ -2109,3 +2117,187 @@ async def remove_release(
|
|
|
2109
2117
|
click.echo(FeedbackManager.success_release_delete(semver=response.get("semver")))
|
|
2110
2118
|
else:
|
|
2111
2119
|
click.echo(FeedbackManager.info_no_release_deleted())
|
|
2120
|
+
|
|
2121
|
+
|
|
2122
|
+
async def validate_aws_iamrole_integration(
|
|
2123
|
+
client: TinyB,
|
|
2124
|
+
service: str,
|
|
2125
|
+
role_arn: Optional[str],
|
|
2126
|
+
region: Optional[str],
|
|
2127
|
+
policy: str = "write",
|
|
2128
|
+
no_validate: Optional[bool] = False,
|
|
2129
|
+
):
|
|
2130
|
+
if no_validate is False:
|
|
2131
|
+
access_policy, trust_policy, external_id = await get_aws_iamrole_policies(
|
|
2132
|
+
client, service=service, policy=policy
|
|
2133
|
+
)
|
|
2134
|
+
|
|
2135
|
+
if not role_arn:
|
|
2136
|
+
if not click.confirm(
|
|
2137
|
+
FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
|
|
2138
|
+
show_default=False,
|
|
2139
|
+
prompt_suffix="Press y to continue:",
|
|
2140
|
+
):
|
|
2141
|
+
sys.exit(1)
|
|
2142
|
+
|
|
2143
|
+
access_policy_copied = True
|
|
2144
|
+
try:
|
|
2145
|
+
pyperclip.copy(access_policy)
|
|
2146
|
+
except Exception:
|
|
2147
|
+
access_policy_copied = False
|
|
2148
|
+
|
|
2149
|
+
replacements_dict = {
|
|
2150
|
+
"<bucket>": "<bucket> with your bucket name",
|
|
2151
|
+
"<table_name>": "<table_name> with your DynamoDB table name",
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
replacements = [
|
|
2155
|
+
replacements_dict.get(replacement, "")
|
|
2156
|
+
for replacement in replacements_dict.keys()
|
|
2157
|
+
if replacement in access_policy
|
|
2158
|
+
]
|
|
2159
|
+
|
|
2160
|
+
if not click.confirm(
|
|
2161
|
+
(
|
|
2162
|
+
FeedbackManager.prompt_s3_iamrole_connection_policy(
|
|
2163
|
+
access_policy=access_policy, replacements=", ".join(replacements)
|
|
2164
|
+
)
|
|
2165
|
+
if access_policy_copied
|
|
2166
|
+
else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
|
|
2167
|
+
),
|
|
2168
|
+
show_default=False,
|
|
2169
|
+
prompt_suffix="Press y to continue:",
|
|
2170
|
+
):
|
|
2171
|
+
sys.exit(1)
|
|
2172
|
+
|
|
2173
|
+
trust_policy_copied = True
|
|
2174
|
+
try:
|
|
2175
|
+
pyperclip.copy(trust_policy)
|
|
2176
|
+
except Exception:
|
|
2177
|
+
trust_policy_copied = False
|
|
2178
|
+
|
|
2179
|
+
if not click.confirm(
|
|
2180
|
+
(
|
|
2181
|
+
FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
|
|
2182
|
+
if trust_policy_copied
|
|
2183
|
+
else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
|
|
2184
|
+
),
|
|
2185
|
+
show_default=False,
|
|
2186
|
+
prompt_suffix="Press y to continue:",
|
|
2187
|
+
):
|
|
2188
|
+
sys.exit(1)
|
|
2189
|
+
else:
|
|
2190
|
+
try:
|
|
2191
|
+
trust_policy = await client.get_trust_policy(service)
|
|
2192
|
+
external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
2193
|
+
except Exception:
|
|
2194
|
+
external_id = ""
|
|
2195
|
+
|
|
2196
|
+
if not role_arn:
|
|
2197
|
+
role_arn = click.prompt("Enter the ARN of the role you just created")
|
|
2198
|
+
validate_string_connector_param("Role ARN", role_arn)
|
|
2199
|
+
|
|
2200
|
+
if not region:
|
|
2201
|
+
region_resource = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
|
|
2202
|
+
region = click.prompt(f"Enter the region where the {region_resource} is located")
|
|
2203
|
+
validate_string_connector_param("Region", region)
|
|
2204
|
+
|
|
2205
|
+
return role_arn, region, external_id
|
|
2206
|
+
|
|
2207
|
+
|
|
2208
|
+
async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "write"):
|
|
2209
|
+
access_policy: Dict[str, Any] = {}
|
|
2210
|
+
if service == DataConnectorType.AMAZON_S3_IAMROLE:
|
|
2211
|
+
service = DataConnectorType.AMAZON_S3
|
|
2212
|
+
try:
|
|
2213
|
+
if policy == "write":
|
|
2214
|
+
access_policy = await client.get_access_write_policy(service)
|
|
2215
|
+
elif policy == "read":
|
|
2216
|
+
access_policy = await client.get_access_read_policy(service)
|
|
2217
|
+
else:
|
|
2218
|
+
raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
|
|
2219
|
+
if not len(access_policy) > 0:
|
|
2220
|
+
raise Exception(f"{service.upper()} Integration not supported in this region")
|
|
2221
|
+
except Exception as e:
|
|
2222
|
+
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
2223
|
+
|
|
2224
|
+
trust_policy: Dict[str, Any] = {}
|
|
2225
|
+
try:
|
|
2226
|
+
trust_policy = await client.get_trust_policy(service)
|
|
2227
|
+
if not len(trust_policy) > 0:
|
|
2228
|
+
raise Exception(f"{service.upper()} Integration not supported in this region")
|
|
2229
|
+
except Exception as e:
|
|
2230
|
+
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
2231
|
+
try:
|
|
2232
|
+
external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
2233
|
+
except Exception:
|
|
2234
|
+
external_id = ""
|
|
2235
|
+
return json.dumps(access_policy, indent=4), json.dumps(trust_policy, indent=4), external_id
|
|
2236
|
+
|
|
2237
|
+
|
|
2238
|
+
async def validate_aws_iamrole_connection_name(
|
|
2239
|
+
client: TinyB, connection_name: Optional[str], no_validate: Optional[bool] = False
|
|
2240
|
+
) -> str:
|
|
2241
|
+
if connection_name and no_validate is False:
|
|
2242
|
+
if await client.get_connector(connection_name) is not None:
|
|
2243
|
+
raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
2244
|
+
else:
|
|
2245
|
+
while not connection_name:
|
|
2246
|
+
connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
|
|
2247
|
+
assert isinstance(connection_name, str)
|
|
2248
|
+
|
|
2249
|
+
if no_validate is False:
|
|
2250
|
+
if await client.get_connector(connection_name) is not None:
|
|
2251
|
+
click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
2252
|
+
connection_name = None
|
|
2253
|
+
assert isinstance(connection_name, str)
|
|
2254
|
+
return connection_name
|
|
2255
|
+
|
|
2256
|
+
|
|
2257
|
+
class DataConnectorType(str, Enum):
|
|
2258
|
+
KAFKA = "kafka"
|
|
2259
|
+
GCLOUD_SCHEDULER = "gcscheduler"
|
|
2260
|
+
SNOWFLAKE = "snowflake"
|
|
2261
|
+
BIGQUERY = "bigquery"
|
|
2262
|
+
GCLOUD_STORAGE = "gcs"
|
|
2263
|
+
GCLOUD_STORAGE_HMAC = "gcs_hmac"
|
|
2264
|
+
GCLOUD_STORAGE_SA = "gcs_service_account"
|
|
2265
|
+
AMAZON_S3 = "s3"
|
|
2266
|
+
AMAZON_S3_IAMROLE = "s3_iamrole"
|
|
2267
|
+
AMAZON_DYNAMODB = "dynamodb"
|
|
2268
|
+
|
|
2269
|
+
def __str__(self) -> str:
|
|
2270
|
+
return self.value
|
|
2271
|
+
|
|
2272
|
+
|
|
2273
|
+
async def create_aws_iamrole_connection(client: TinyB, service: str, connection_name, role_arn, region) -> None:
|
|
2274
|
+
conn_file_name = f"{connection_name}.connection"
|
|
2275
|
+
conn_file_path = Path(getcwd(), conn_file_name)
|
|
2276
|
+
|
|
2277
|
+
if os.path.isfile(conn_file_path):
|
|
2278
|
+
raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))
|
|
2279
|
+
|
|
2280
|
+
if service == DataConnectorType.AMAZON_S3_IAMROLE:
|
|
2281
|
+
click.echo(FeedbackManager.info_creating_s3_iamrole_connection(connection_name=connection_name))
|
|
2282
|
+
if service == DataConnectorType.AMAZON_DYNAMODB:
|
|
2283
|
+
click.echo(FeedbackManager.info_creating_dynamodb_connection(connection_name=connection_name))
|
|
2284
|
+
|
|
2285
|
+
params = ConnectionReplacements.map_api_params_from_prompt_params(
|
|
2286
|
+
service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
2287
|
+
)
|
|
2288
|
+
|
|
2289
|
+
click.echo("** Creating connection...")
|
|
2290
|
+
try:
|
|
2291
|
+
_ = await client.connection_create(params)
|
|
2292
|
+
except Exception as e:
|
|
2293
|
+
raise CLIConnectionException(
|
|
2294
|
+
FeedbackManager.error_connection_create(connection_name=connection_name, error=str(e))
|
|
2295
|
+
)
|
|
2296
|
+
|
|
2297
|
+
with open(conn_file_path, "w") as f:
|
|
2298
|
+
f.write(
|
|
2299
|
+
f"""TYPE {service}
|
|
2300
|
+
|
|
2301
|
+
"""
|
|
2302
|
+
)
|
|
2303
|
+
click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
|
|
@@ -3,16 +3,12 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import json
|
|
7
6
|
import os
|
|
8
|
-
import sys
|
|
9
|
-
from enum import Enum
|
|
10
7
|
from os import getcwd
|
|
11
8
|
from pathlib import Path
|
|
12
9
|
from typing import Any, Dict, List, Optional
|
|
13
10
|
|
|
14
11
|
import click
|
|
15
|
-
import pyperclip
|
|
16
12
|
from click import Context
|
|
17
13
|
|
|
18
14
|
from tinybird.client import DoesNotExistException, TinyB
|
|
@@ -20,9 +16,13 @@ from tinybird.feedback_manager import FeedbackManager
|
|
|
20
16
|
from tinybird.tb_cli_modules.cli import cli
|
|
21
17
|
from tinybird.tb_cli_modules.common import (
|
|
22
18
|
ConnectionReplacements,
|
|
19
|
+
DataConnectorType,
|
|
23
20
|
_get_setting_value,
|
|
24
21
|
coro,
|
|
22
|
+
create_aws_iamrole_connection,
|
|
25
23
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
24
|
+
validate_aws_iamrole_connection_name,
|
|
25
|
+
validate_aws_iamrole_integration,
|
|
26
26
|
validate_connection_name,
|
|
27
27
|
validate_kafka_auto_offset_reset,
|
|
28
28
|
validate_kafka_bootstrap_servers,
|
|
@@ -34,22 +34,6 @@ from tinybird.tb_cli_modules.common import (
|
|
|
34
34
|
from tinybird.tb_cli_modules.exceptions import CLIConnectionException
|
|
35
35
|
from tinybird.tb_cli_modules.telemetry import is_ci_environment
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
class DataConnectorType(str, Enum):
|
|
39
|
-
KAFKA = "kafka"
|
|
40
|
-
GCLOUD_SCHEDULER = "gcscheduler"
|
|
41
|
-
SNOWFLAKE = "snowflake"
|
|
42
|
-
BIGQUERY = "bigquery"
|
|
43
|
-
GCLOUD_STORAGE = "gcs"
|
|
44
|
-
GCLOUD_STORAGE_HMAC = "gcs_hmac"
|
|
45
|
-
GCLOUD_STORAGE_SA = "gcs_service_account"
|
|
46
|
-
AMAZON_S3 = "s3"
|
|
47
|
-
AMAZON_S3_IAMROLE = "s3_iamrole"
|
|
48
|
-
|
|
49
|
-
def __str__(self) -> str:
|
|
50
|
-
return self.value
|
|
51
|
-
|
|
52
|
-
|
|
53
37
|
DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
54
38
|
DataConnectorType.KAFKA: [
|
|
55
39
|
"kafka_bootstrap_servers",
|
|
@@ -96,6 +80,11 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
|
96
80
|
"s3_iamrole_region",
|
|
97
81
|
"s3_iamrole_external_id",
|
|
98
82
|
],
|
|
83
|
+
DataConnectorType.AMAZON_DYNAMODB: [
|
|
84
|
+
"dynamodb_iamrole_arn",
|
|
85
|
+
"dynamodb_iamrole_region",
|
|
86
|
+
"dynamodb_iamrole_external_id",
|
|
87
|
+
],
|
|
99
88
|
}
|
|
100
89
|
|
|
101
90
|
SENSITIVE_CONNECTOR_SETTINGS = {
|
|
@@ -108,6 +97,7 @@ SENSITIVE_CONNECTOR_SETTINGS = {
|
|
|
108
97
|
DataConnectorType.GCLOUD_STORAGE_HMAC: ["gcs_hmac_secret"],
|
|
109
98
|
DataConnectorType.AMAZON_S3: ["s3_secret_access_key"],
|
|
110
99
|
DataConnectorType.AMAZON_S3_IAMROLE: ["s3_iamrole_arn"],
|
|
100
|
+
DataConnectorType.AMAZON_DYNAMODB: ["dynamodb_iamrole_arn"],
|
|
111
101
|
}
|
|
112
102
|
|
|
113
103
|
|
|
@@ -735,147 +725,68 @@ async def connection_create_s3_iamrole(
|
|
|
735
725
|
|
|
736
726
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
737
727
|
client: TinyB = obj["client"]
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
752
|
-
|
|
753
|
-
s3_trust_policy: Dict[str, Any] = {}
|
|
754
|
-
try:
|
|
755
|
-
s3_trust_policy = await client.get_s3_trust_policy()
|
|
756
|
-
if not len(s3_trust_policy) > 0:
|
|
757
|
-
raise Exception("S3 Integration not supported in this region")
|
|
758
|
-
except Exception as e:
|
|
759
|
-
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
760
|
-
try:
|
|
761
|
-
external_id = s3_trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
762
|
-
except Exception:
|
|
763
|
-
external_id = ""
|
|
764
|
-
return json.dumps(s3_access_policy, indent=4), json.dumps(s3_trust_policy, indent=4), external_id
|
|
765
|
-
|
|
766
|
-
async def validate_s3_integration(role_arn: Optional[str], region: Optional[str]):
|
|
767
|
-
if no_validate is False:
|
|
768
|
-
access_policy, trust_policy, external_id = await get_s3_policies()
|
|
769
|
-
|
|
770
|
-
if not role_arn:
|
|
771
|
-
if not click.confirm(
|
|
772
|
-
FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
|
|
773
|
-
show_default=False,
|
|
774
|
-
prompt_suffix="Press y to continue:",
|
|
775
|
-
):
|
|
776
|
-
sys.exit(1)
|
|
777
|
-
|
|
778
|
-
access_policy_copied = True
|
|
779
|
-
try:
|
|
780
|
-
pyperclip.copy(access_policy)
|
|
781
|
-
except Exception:
|
|
782
|
-
access_policy_copied = False
|
|
783
|
-
|
|
784
|
-
if not click.confirm(
|
|
785
|
-
(
|
|
786
|
-
FeedbackManager.prompt_s3_iamrole_connection_policy(access_policy=access_policy)
|
|
787
|
-
if access_policy_copied
|
|
788
|
-
else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
|
|
789
|
-
),
|
|
790
|
-
show_default=False,
|
|
791
|
-
prompt_suffix="Press y to continue:",
|
|
792
|
-
):
|
|
793
|
-
sys.exit(1)
|
|
794
|
-
|
|
795
|
-
trust_policy_copied = True
|
|
796
|
-
try:
|
|
797
|
-
pyperclip.copy(trust_policy)
|
|
798
|
-
except Exception:
|
|
799
|
-
trust_policy_copied = False
|
|
800
|
-
|
|
801
|
-
if not click.confirm(
|
|
802
|
-
(
|
|
803
|
-
FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
|
|
804
|
-
if trust_policy_copied
|
|
805
|
-
else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
|
|
806
|
-
),
|
|
807
|
-
show_default=False,
|
|
808
|
-
prompt_suffix="Press y to continue:",
|
|
809
|
-
):
|
|
810
|
-
sys.exit(1)
|
|
811
|
-
else:
|
|
812
|
-
try:
|
|
813
|
-
trust_policy = await client.get_s3_trust_policy()
|
|
814
|
-
external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
815
|
-
except Exception:
|
|
816
|
-
external_id = ""
|
|
817
|
-
if not role_arn:
|
|
818
|
-
role_arn = click.prompt("Enter the ARN of the role you just created")
|
|
819
|
-
validate_string_connector_param("Role ARN", role_arn)
|
|
820
|
-
|
|
821
|
-
if not region:
|
|
822
|
-
region = click.prompt("Enter the region where the bucket is located")
|
|
823
|
-
validate_string_connector_param("Region", region)
|
|
824
|
-
|
|
825
|
-
return role_arn, region, external_id
|
|
826
|
-
|
|
827
|
-
async def validate_connection_name(connection_name: Optional[str]) -> str:
|
|
828
|
-
if connection_name and no_validate is False:
|
|
829
|
-
if await client.get_connector(connection_name) is not None:
|
|
830
|
-
raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
831
|
-
else:
|
|
832
|
-
while not connection_name:
|
|
833
|
-
connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
|
|
834
|
-
assert isinstance(connection_name, str)
|
|
835
|
-
|
|
836
|
-
if no_validate is False:
|
|
837
|
-
if await client.get_connector(connection_name) is not None:
|
|
838
|
-
click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
839
|
-
connection_name = None
|
|
840
|
-
assert isinstance(connection_name, str)
|
|
841
|
-
return connection_name
|
|
842
|
-
|
|
843
|
-
async def create_connection(connection_name, role_arn, region) -> None:
|
|
844
|
-
conn_file_name = f"{connection_name}.connection"
|
|
845
|
-
conn_file_path = Path(getcwd(), conn_file_name)
|
|
846
|
-
service = DataConnectorType.AMAZON_S3_IAMROLE
|
|
847
|
-
|
|
848
|
-
if os.path.isfile(conn_file_path):
|
|
849
|
-
raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))
|
|
850
|
-
|
|
851
|
-
click.echo(FeedbackManager.info_creating_s3_iamrole_connection(connection_name=connection_name))
|
|
852
|
-
|
|
853
|
-
params = ConnectionReplacements.map_api_params_from_prompt_params(
|
|
854
|
-
service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
855
|
-
)
|
|
856
|
-
|
|
857
|
-
click.echo("** Creating connection...")
|
|
858
|
-
try:
|
|
859
|
-
_ = await client.connection_create(params)
|
|
860
|
-
except Exception as e:
|
|
861
|
-
raise CLIConnectionException(
|
|
862
|
-
FeedbackManager.error_connection_create(connection_name=connection_name, error=str(e))
|
|
863
|
-
)
|
|
864
|
-
|
|
865
|
-
with open(conn_file_path, "w") as f:
|
|
866
|
-
f.write(
|
|
867
|
-
f"""TYPE {service}
|
|
868
|
-
|
|
869
|
-
"""
|
|
870
|
-
)
|
|
871
|
-
click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
|
|
872
|
-
|
|
873
|
-
role_arn, region, external_id = await validate_s3_integration(role_arn, region)
|
|
874
|
-
connection_name = await validate_connection_name(connection_name)
|
|
875
|
-
await create_connection(connection_name, role_arn, region)
|
|
728
|
+
service = DataConnectorType.AMAZON_S3_IAMROLE
|
|
729
|
+
role_arn, region, external_id = await validate_aws_iamrole_integration(
|
|
730
|
+
client,
|
|
731
|
+
service=service,
|
|
732
|
+
role_arn=role_arn,
|
|
733
|
+
region=region,
|
|
734
|
+
policy=policy,
|
|
735
|
+
no_validate=no_validate,
|
|
736
|
+
)
|
|
737
|
+
connection_name = await validate_aws_iamrole_connection_name(client, connection_name, no_validate)
|
|
738
|
+
await create_aws_iamrole_connection(
|
|
739
|
+
client, service=service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
740
|
+
)
|
|
876
741
|
if external_id:
|
|
877
742
|
click.echo(
|
|
878
743
|
FeedbackManager.success_s3_iam_connection_created(
|
|
879
744
|
connection_name=connection_name, external_id=external_id, role_arn=role_arn
|
|
880
745
|
)
|
|
881
746
|
)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
@connection_create.command(
|
|
750
|
+
name="dynamodb", short_help="Creates a AWS DynamoDB connection using IAM role authentication", hidden=True
|
|
751
|
+
)
|
|
752
|
+
@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
|
|
753
|
+
@click.option("--role-arn", default=None, help="The ARN of the IAM role to use for the connection")
|
|
754
|
+
@click.option("--region", default=None, help="The AWS region where DynamoDB is located")
|
|
755
|
+
@click.option("--no-validate", is_flag=True, default=False, help="Do not validate DynamoDB connection during creation")
|
|
756
|
+
@click.pass_context
|
|
757
|
+
@coro
|
|
758
|
+
async def connection_create_dynamodb(
|
|
759
|
+
ctx: Context,
|
|
760
|
+
connection_name: Optional[str] = "",
|
|
761
|
+
role_arn: Optional[str] = "",
|
|
762
|
+
region: Optional[str] = "",
|
|
763
|
+
no_validate: Optional[bool] = False,
|
|
764
|
+
) -> None:
|
|
765
|
+
"""
|
|
766
|
+
Creates a DynamoDB connection using IAM role authentication in the current workspace
|
|
767
|
+
|
|
768
|
+
\b
|
|
769
|
+
$ tb connection create dynamodb
|
|
770
|
+
"""
|
|
771
|
+
|
|
772
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
773
|
+
client: TinyB = obj["client"]
|
|
774
|
+
|
|
775
|
+
service = DataConnectorType.AMAZON_DYNAMODB
|
|
776
|
+
role_arn, region, _external_id = await validate_aws_iamrole_integration(
|
|
777
|
+
client,
|
|
778
|
+
service=service,
|
|
779
|
+
role_arn=role_arn,
|
|
780
|
+
region=region,
|
|
781
|
+
policy="read",
|
|
782
|
+
no_validate=no_validate,
|
|
783
|
+
)
|
|
784
|
+
connection_name = await validate_aws_iamrole_connection_name(client, connection_name, no_validate)
|
|
785
|
+
await create_aws_iamrole_connection(
|
|
786
|
+
client, service=service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
787
|
+
)
|
|
788
|
+
click.echo(
|
|
789
|
+
FeedbackManager.success_dynamodb_connection_created(
|
|
790
|
+
connection_name=connection_name, region=region, role_arn=role_arn
|
|
791
|
+
)
|
|
792
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.2.2.
|
|
3
|
+
Version: 5.2.2.dev3
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,8 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
+
|
|
22
|
+
5.2.2.dev3
|
|
23
|
+
**********
|
|
24
|
+
|
|
25
|
+
- `Added` `tb connection create` now supports `dynamodb` as service type
|
|
26
|
+
|
|
27
|
+
5.2.2.dev2
|
|
28
|
+
**********
|
|
29
|
+
|
|
30
|
+
- `Added` error when trying to push a data source with `SETTINGS` instead of `ENGINE_SETTINGS`
|
|
31
|
+
|
|
21
32
|
5.2.2.dev1
|
|
22
33
|
**********
|
|
34
|
+
|
|
23
35
|
- `Added` support for buckets with gzip files when creating S3 Data Sources
|
|
24
36
|
|
|
25
37
|
5.2.1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py
RENAMED
|
File without changes
|
{tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/workspace_members.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.2.2.dev1 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|