tinybird-cli 5.2.2.dev2__tar.gz → 5.2.2.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/PKG-INFO +9 -1
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/__cli__.py +2 -2
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/client.py +6 -6
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/feedback_manager.py +5 -2
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/common.py +192 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/connection.py +69 -158
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/PKG-INFO +9 -1
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/setup.cfg +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/check_pypi.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/config.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/connectors.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/context.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/datafile.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/datatypes.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/git_settings.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/sql.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/sql_template.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/sql_toolset.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/syncasync.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/token.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tornado_template.py +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/SOURCES.txt +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/dependency_links.txt +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/entry_points.txt +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/requires.txt +0 -0
- {tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.2.2.
|
|
3
|
+
Version: 5.2.2.dev3
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,12 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
+
|
|
22
|
+
5.2.2.dev3
|
|
23
|
+
**********
|
|
24
|
+
|
|
25
|
+
- `Added` `tb connection create` now supports `dynamodb` as service type
|
|
26
|
+
|
|
21
27
|
5.2.2.dev2
|
|
22
28
|
**********
|
|
29
|
+
|
|
23
30
|
- `Added` error when trying to push a data source with `SETTINGS` instead of `ENGINE_SETTINGS`
|
|
24
31
|
|
|
25
32
|
5.2.2.dev1
|
|
26
33
|
**********
|
|
34
|
+
|
|
27
35
|
- `Added` support for buckets with gzip files when creating S3 Data Sources
|
|
28
36
|
|
|
29
37
|
5.2.1
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '5.2.2.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '5.2.2.dev3'
|
|
8
|
+
__revision__ = 'decfb20'
|
|
@@ -1011,14 +1011,14 @@ class TinyB(object):
|
|
|
1011
1011
|
response = await self._req(f"/v0/connectors/snowflake/warehouses?{urlencode(params)}", method="POST", data="")
|
|
1012
1012
|
return response["warehouses"]
|
|
1013
1013
|
|
|
1014
|
-
async def
|
|
1015
|
-
return await self._req("/v0/integrations/
|
|
1014
|
+
async def get_trust_policy(self, service: str) -> Dict[str, Any]:
|
|
1015
|
+
return await self._req(f"/v0/integrations/{service}/policies/trust-policy")
|
|
1016
1016
|
|
|
1017
|
-
async def
|
|
1018
|
-
return await self._req("/v0/integrations/
|
|
1017
|
+
async def get_access_write_policy(self, service: str) -> Dict[str, Any]:
|
|
1018
|
+
return await self._req(f"/v0/integrations/{service}/policies/write-access-policy")
|
|
1019
1019
|
|
|
1020
|
-
async def
|
|
1021
|
-
return await self._req("/v0/integrations/
|
|
1020
|
+
async def get_access_read_policy(self, service: str) -> Dict[str, Any]:
|
|
1021
|
+
return await self._req(f"/v0/integrations/{service}/policies/read-access-policy")
|
|
1022
1022
|
|
|
1023
1023
|
async def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
|
1024
1024
|
try:
|
|
@@ -431,7 +431,7 @@ Ready? """
|
|
|
431
431
|
|
|
432
432
|
prompt_s3_iamrole_connection_login_aws = prompt_message("""[1] Log into your AWS Console\n\n""")
|
|
433
433
|
prompt_s3_iamrole_connection_policy = prompt_message(
|
|
434
|
-
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace
|
|
434
|
+
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, replace {replacements}:\n\n{access_policy}\n\n(The policy has been copied to your clipboard)\n\n"""
|
|
435
435
|
)
|
|
436
436
|
prompt_s3_iamrole_connection_policy_not_copied = prompt_message(
|
|
437
437
|
"""\n[2] Go to IAM > Policies. Create a new policy with the following permissions. Please, copy this policy and replace <bucket> with your bucket name:\n\n{access_policy}\n\n"""
|
|
@@ -476,6 +476,7 @@ Ready? """
|
|
|
476
476
|
)
|
|
477
477
|
info_creating_kafka_connection = info_message("** Creating new Kafka connection '{connection_name}'")
|
|
478
478
|
info_creating_s3_iamrole_connection = info_message("** Creating new S3 IAM Role connection '{connection_name}'")
|
|
479
|
+
info_creating_dynamodb_connection = info_message("** Creating new DynamoDB connection '{connection_name}'")
|
|
479
480
|
|
|
480
481
|
warning_remove_oldest_rollback = warning_message(
|
|
481
482
|
"** [WARNING] Will try to remove oldest rollback Release before promoting to live Release {semver}."
|
|
@@ -910,7 +911,9 @@ Ready? """
|
|
|
910
911
|
success_s3_iam_connection_created = success_message(
|
|
911
912
|
"** Info associated with this connection:\n** External ID: {external_id}\n** Role ARN: {role_arn}"
|
|
912
913
|
)
|
|
913
|
-
|
|
914
|
+
success_dynamodb_connection_created = success_message(
|
|
915
|
+
"** Info associated with this DynamoDB connection:\n** Region: {region}\n** Role ARN: {role_arn}"
|
|
916
|
+
)
|
|
914
917
|
success_delete_connection = success_message("** Connection {connection_id} removed successfully")
|
|
915
918
|
success_connection_using = success_message("** Using connection '{connection_name}'")
|
|
916
919
|
success_using_host = success_message("** Using host: {host} ({name})")
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import asyncio
|
|
9
9
|
import json
|
|
10
|
+
import os
|
|
10
11
|
import re
|
|
11
12
|
import socket
|
|
12
13
|
import sys
|
|
@@ -24,6 +25,7 @@ import click
|
|
|
24
25
|
import click.formatting
|
|
25
26
|
import humanfriendly
|
|
26
27
|
import humanfriendly.tables
|
|
28
|
+
import pyperclip
|
|
27
29
|
from click import Context
|
|
28
30
|
from click._termui_impl import ProgressBar
|
|
29
31
|
from humanfriendly.tables import format_pretty_table
|
|
@@ -1659,6 +1661,12 @@ class ConnectionReplacements:
|
|
|
1659
1661
|
"private_key_id": "gcs_private_key_id",
|
|
1660
1662
|
"connection_name": "name",
|
|
1661
1663
|
},
|
|
1664
|
+
"dynamodb": {
|
|
1665
|
+
"service": "service",
|
|
1666
|
+
"connection_name": "name",
|
|
1667
|
+
"role_arn": "dynamodb_iamrole_arn",
|
|
1668
|
+
"region": "dynamodb_iamrole_region",
|
|
1669
|
+
},
|
|
1662
1670
|
}
|
|
1663
1671
|
|
|
1664
1672
|
@staticmethod
|
|
@@ -2109,3 +2117,187 @@ async def remove_release(
|
|
|
2109
2117
|
click.echo(FeedbackManager.success_release_delete(semver=response.get("semver")))
|
|
2110
2118
|
else:
|
|
2111
2119
|
click.echo(FeedbackManager.info_no_release_deleted())
|
|
2120
|
+
|
|
2121
|
+
|
|
2122
|
+
async def validate_aws_iamrole_integration(
|
|
2123
|
+
client: TinyB,
|
|
2124
|
+
service: str,
|
|
2125
|
+
role_arn: Optional[str],
|
|
2126
|
+
region: Optional[str],
|
|
2127
|
+
policy: str = "write",
|
|
2128
|
+
no_validate: Optional[bool] = False,
|
|
2129
|
+
):
|
|
2130
|
+
if no_validate is False:
|
|
2131
|
+
access_policy, trust_policy, external_id = await get_aws_iamrole_policies(
|
|
2132
|
+
client, service=service, policy=policy
|
|
2133
|
+
)
|
|
2134
|
+
|
|
2135
|
+
if not role_arn:
|
|
2136
|
+
if not click.confirm(
|
|
2137
|
+
FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
|
|
2138
|
+
show_default=False,
|
|
2139
|
+
prompt_suffix="Press y to continue:",
|
|
2140
|
+
):
|
|
2141
|
+
sys.exit(1)
|
|
2142
|
+
|
|
2143
|
+
access_policy_copied = True
|
|
2144
|
+
try:
|
|
2145
|
+
pyperclip.copy(access_policy)
|
|
2146
|
+
except Exception:
|
|
2147
|
+
access_policy_copied = False
|
|
2148
|
+
|
|
2149
|
+
replacements_dict = {
|
|
2150
|
+
"<bucket>": "<bucket> with your bucket name",
|
|
2151
|
+
"<table_name>": "<table_name> with your DynamoDB table name",
|
|
2152
|
+
}
|
|
2153
|
+
|
|
2154
|
+
replacements = [
|
|
2155
|
+
replacements_dict.get(replacement, "")
|
|
2156
|
+
for replacement in replacements_dict.keys()
|
|
2157
|
+
if replacement in access_policy
|
|
2158
|
+
]
|
|
2159
|
+
|
|
2160
|
+
if not click.confirm(
|
|
2161
|
+
(
|
|
2162
|
+
FeedbackManager.prompt_s3_iamrole_connection_policy(
|
|
2163
|
+
access_policy=access_policy, replacements=", ".join(replacements)
|
|
2164
|
+
)
|
|
2165
|
+
if access_policy_copied
|
|
2166
|
+
else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
|
|
2167
|
+
),
|
|
2168
|
+
show_default=False,
|
|
2169
|
+
prompt_suffix="Press y to continue:",
|
|
2170
|
+
):
|
|
2171
|
+
sys.exit(1)
|
|
2172
|
+
|
|
2173
|
+
trust_policy_copied = True
|
|
2174
|
+
try:
|
|
2175
|
+
pyperclip.copy(trust_policy)
|
|
2176
|
+
except Exception:
|
|
2177
|
+
trust_policy_copied = False
|
|
2178
|
+
|
|
2179
|
+
if not click.confirm(
|
|
2180
|
+
(
|
|
2181
|
+
FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
|
|
2182
|
+
if trust_policy_copied
|
|
2183
|
+
else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
|
|
2184
|
+
),
|
|
2185
|
+
show_default=False,
|
|
2186
|
+
prompt_suffix="Press y to continue:",
|
|
2187
|
+
):
|
|
2188
|
+
sys.exit(1)
|
|
2189
|
+
else:
|
|
2190
|
+
try:
|
|
2191
|
+
trust_policy = await client.get_trust_policy(service)
|
|
2192
|
+
external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
2193
|
+
except Exception:
|
|
2194
|
+
external_id = ""
|
|
2195
|
+
|
|
2196
|
+
if not role_arn:
|
|
2197
|
+
role_arn = click.prompt("Enter the ARN of the role you just created")
|
|
2198
|
+
validate_string_connector_param("Role ARN", role_arn)
|
|
2199
|
+
|
|
2200
|
+
if not region:
|
|
2201
|
+
region_resource = "table" if service == DataConnectorType.AMAZON_DYNAMODB else "bucket"
|
|
2202
|
+
region = click.prompt(f"Enter the region where the {region_resource} is located")
|
|
2203
|
+
validate_string_connector_param("Region", region)
|
|
2204
|
+
|
|
2205
|
+
return role_arn, region, external_id
|
|
2206
|
+
|
|
2207
|
+
|
|
2208
|
+
async def get_aws_iamrole_policies(client: TinyB, service: str, policy: str = "write"):
|
|
2209
|
+
access_policy: Dict[str, Any] = {}
|
|
2210
|
+
if service == DataConnectorType.AMAZON_S3_IAMROLE:
|
|
2211
|
+
service = DataConnectorType.AMAZON_S3
|
|
2212
|
+
try:
|
|
2213
|
+
if policy == "write":
|
|
2214
|
+
access_policy = await client.get_access_write_policy(service)
|
|
2215
|
+
elif policy == "read":
|
|
2216
|
+
access_policy = await client.get_access_read_policy(service)
|
|
2217
|
+
else:
|
|
2218
|
+
raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
|
|
2219
|
+
if not len(access_policy) > 0:
|
|
2220
|
+
raise Exception(f"{service.upper()} Integration not supported in this region")
|
|
2221
|
+
except Exception as e:
|
|
2222
|
+
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
2223
|
+
|
|
2224
|
+
trust_policy: Dict[str, Any] = {}
|
|
2225
|
+
try:
|
|
2226
|
+
trust_policy = await client.get_trust_policy(service)
|
|
2227
|
+
if not len(trust_policy) > 0:
|
|
2228
|
+
raise Exception(f"{service.upper()} Integration not supported in this region")
|
|
2229
|
+
except Exception as e:
|
|
2230
|
+
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
2231
|
+
try:
|
|
2232
|
+
external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
2233
|
+
except Exception:
|
|
2234
|
+
external_id = ""
|
|
2235
|
+
return json.dumps(access_policy, indent=4), json.dumps(trust_policy, indent=4), external_id
|
|
2236
|
+
|
|
2237
|
+
|
|
2238
|
+
async def validate_aws_iamrole_connection_name(
|
|
2239
|
+
client: TinyB, connection_name: Optional[str], no_validate: Optional[bool] = False
|
|
2240
|
+
) -> str:
|
|
2241
|
+
if connection_name and no_validate is False:
|
|
2242
|
+
if await client.get_connector(connection_name) is not None:
|
|
2243
|
+
raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
2244
|
+
else:
|
|
2245
|
+
while not connection_name:
|
|
2246
|
+
connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
|
|
2247
|
+
assert isinstance(connection_name, str)
|
|
2248
|
+
|
|
2249
|
+
if no_validate is False:
|
|
2250
|
+
if await client.get_connector(connection_name) is not None:
|
|
2251
|
+
click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
2252
|
+
connection_name = None
|
|
2253
|
+
assert isinstance(connection_name, str)
|
|
2254
|
+
return connection_name
|
|
2255
|
+
|
|
2256
|
+
|
|
2257
|
+
class DataConnectorType(str, Enum):
|
|
2258
|
+
KAFKA = "kafka"
|
|
2259
|
+
GCLOUD_SCHEDULER = "gcscheduler"
|
|
2260
|
+
SNOWFLAKE = "snowflake"
|
|
2261
|
+
BIGQUERY = "bigquery"
|
|
2262
|
+
GCLOUD_STORAGE = "gcs"
|
|
2263
|
+
GCLOUD_STORAGE_HMAC = "gcs_hmac"
|
|
2264
|
+
GCLOUD_STORAGE_SA = "gcs_service_account"
|
|
2265
|
+
AMAZON_S3 = "s3"
|
|
2266
|
+
AMAZON_S3_IAMROLE = "s3_iamrole"
|
|
2267
|
+
AMAZON_DYNAMODB = "dynamodb"
|
|
2268
|
+
|
|
2269
|
+
def __str__(self) -> str:
|
|
2270
|
+
return self.value
|
|
2271
|
+
|
|
2272
|
+
|
|
2273
|
+
async def create_aws_iamrole_connection(client: TinyB, service: str, connection_name, role_arn, region) -> None:
|
|
2274
|
+
conn_file_name = f"{connection_name}.connection"
|
|
2275
|
+
conn_file_path = Path(getcwd(), conn_file_name)
|
|
2276
|
+
|
|
2277
|
+
if os.path.isfile(conn_file_path):
|
|
2278
|
+
raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))
|
|
2279
|
+
|
|
2280
|
+
if service == DataConnectorType.AMAZON_S3_IAMROLE:
|
|
2281
|
+
click.echo(FeedbackManager.info_creating_s3_iamrole_connection(connection_name=connection_name))
|
|
2282
|
+
if service == DataConnectorType.AMAZON_DYNAMODB:
|
|
2283
|
+
click.echo(FeedbackManager.info_creating_dynamodb_connection(connection_name=connection_name))
|
|
2284
|
+
|
|
2285
|
+
params = ConnectionReplacements.map_api_params_from_prompt_params(
|
|
2286
|
+
service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
2287
|
+
)
|
|
2288
|
+
|
|
2289
|
+
click.echo("** Creating connection...")
|
|
2290
|
+
try:
|
|
2291
|
+
_ = await client.connection_create(params)
|
|
2292
|
+
except Exception as e:
|
|
2293
|
+
raise CLIConnectionException(
|
|
2294
|
+
FeedbackManager.error_connection_create(connection_name=connection_name, error=str(e))
|
|
2295
|
+
)
|
|
2296
|
+
|
|
2297
|
+
with open(conn_file_path, "w") as f:
|
|
2298
|
+
f.write(
|
|
2299
|
+
f"""TYPE {service}
|
|
2300
|
+
|
|
2301
|
+
"""
|
|
2302
|
+
)
|
|
2303
|
+
click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
|
|
@@ -3,16 +3,12 @@
|
|
|
3
3
|
# - If it makes sense and only when strictly necessary, you can create utility functions in this file.
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
|
-
import json
|
|
7
6
|
import os
|
|
8
|
-
import sys
|
|
9
|
-
from enum import Enum
|
|
10
7
|
from os import getcwd
|
|
11
8
|
from pathlib import Path
|
|
12
9
|
from typing import Any, Dict, List, Optional
|
|
13
10
|
|
|
14
11
|
import click
|
|
15
|
-
import pyperclip
|
|
16
12
|
from click import Context
|
|
17
13
|
|
|
18
14
|
from tinybird.client import DoesNotExistException, TinyB
|
|
@@ -20,9 +16,13 @@ from tinybird.feedback_manager import FeedbackManager
|
|
|
20
16
|
from tinybird.tb_cli_modules.cli import cli
|
|
21
17
|
from tinybird.tb_cli_modules.common import (
|
|
22
18
|
ConnectionReplacements,
|
|
19
|
+
DataConnectorType,
|
|
23
20
|
_get_setting_value,
|
|
24
21
|
coro,
|
|
22
|
+
create_aws_iamrole_connection,
|
|
25
23
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
24
|
+
validate_aws_iamrole_connection_name,
|
|
25
|
+
validate_aws_iamrole_integration,
|
|
26
26
|
validate_connection_name,
|
|
27
27
|
validate_kafka_auto_offset_reset,
|
|
28
28
|
validate_kafka_bootstrap_servers,
|
|
@@ -34,22 +34,6 @@ from tinybird.tb_cli_modules.common import (
|
|
|
34
34
|
from tinybird.tb_cli_modules.exceptions import CLIConnectionException
|
|
35
35
|
from tinybird.tb_cli_modules.telemetry import is_ci_environment
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
class DataConnectorType(str, Enum):
|
|
39
|
-
KAFKA = "kafka"
|
|
40
|
-
GCLOUD_SCHEDULER = "gcscheduler"
|
|
41
|
-
SNOWFLAKE = "snowflake"
|
|
42
|
-
BIGQUERY = "bigquery"
|
|
43
|
-
GCLOUD_STORAGE = "gcs"
|
|
44
|
-
GCLOUD_STORAGE_HMAC = "gcs_hmac"
|
|
45
|
-
GCLOUD_STORAGE_SA = "gcs_service_account"
|
|
46
|
-
AMAZON_S3 = "s3"
|
|
47
|
-
AMAZON_S3_IAMROLE = "s3_iamrole"
|
|
48
|
-
|
|
49
|
-
def __str__(self) -> str:
|
|
50
|
-
return self.value
|
|
51
|
-
|
|
52
|
-
|
|
53
37
|
DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
54
38
|
DataConnectorType.KAFKA: [
|
|
55
39
|
"kafka_bootstrap_servers",
|
|
@@ -96,6 +80,11 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
|
96
80
|
"s3_iamrole_region",
|
|
97
81
|
"s3_iamrole_external_id",
|
|
98
82
|
],
|
|
83
|
+
DataConnectorType.AMAZON_DYNAMODB: [
|
|
84
|
+
"dynamodb_iamrole_arn",
|
|
85
|
+
"dynamodb_iamrole_region",
|
|
86
|
+
"dynamodb_iamrole_external_id",
|
|
87
|
+
],
|
|
99
88
|
}
|
|
100
89
|
|
|
101
90
|
SENSITIVE_CONNECTOR_SETTINGS = {
|
|
@@ -108,6 +97,7 @@ SENSITIVE_CONNECTOR_SETTINGS = {
|
|
|
108
97
|
DataConnectorType.GCLOUD_STORAGE_HMAC: ["gcs_hmac_secret"],
|
|
109
98
|
DataConnectorType.AMAZON_S3: ["s3_secret_access_key"],
|
|
110
99
|
DataConnectorType.AMAZON_S3_IAMROLE: ["s3_iamrole_arn"],
|
|
100
|
+
DataConnectorType.AMAZON_DYNAMODB: ["dynamodb_iamrole_arn"],
|
|
111
101
|
}
|
|
112
102
|
|
|
113
103
|
|
|
@@ -735,147 +725,68 @@ async def connection_create_s3_iamrole(
|
|
|
735
725
|
|
|
736
726
|
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
737
727
|
client: TinyB = obj["client"]
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
752
|
-
|
|
753
|
-
s3_trust_policy: Dict[str, Any] = {}
|
|
754
|
-
try:
|
|
755
|
-
s3_trust_policy = await client.get_s3_trust_policy()
|
|
756
|
-
if not len(s3_trust_policy) > 0:
|
|
757
|
-
raise Exception("S3 Integration not supported in this region")
|
|
758
|
-
except Exception as e:
|
|
759
|
-
raise CLIConnectionException(FeedbackManager.error_connection_integration_not_available(error=str(e)))
|
|
760
|
-
try:
|
|
761
|
-
external_id = s3_trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
762
|
-
except Exception:
|
|
763
|
-
external_id = ""
|
|
764
|
-
return json.dumps(s3_access_policy, indent=4), json.dumps(s3_trust_policy, indent=4), external_id
|
|
765
|
-
|
|
766
|
-
async def validate_s3_integration(role_arn: Optional[str], region: Optional[str]):
|
|
767
|
-
if no_validate is False:
|
|
768
|
-
access_policy, trust_policy, external_id = await get_s3_policies()
|
|
769
|
-
|
|
770
|
-
if not role_arn:
|
|
771
|
-
if not click.confirm(
|
|
772
|
-
FeedbackManager.prompt_s3_iamrole_connection_login_aws(),
|
|
773
|
-
show_default=False,
|
|
774
|
-
prompt_suffix="Press y to continue:",
|
|
775
|
-
):
|
|
776
|
-
sys.exit(1)
|
|
777
|
-
|
|
778
|
-
access_policy_copied = True
|
|
779
|
-
try:
|
|
780
|
-
pyperclip.copy(access_policy)
|
|
781
|
-
except Exception:
|
|
782
|
-
access_policy_copied = False
|
|
783
|
-
|
|
784
|
-
if not click.confirm(
|
|
785
|
-
(
|
|
786
|
-
FeedbackManager.prompt_s3_iamrole_connection_policy(access_policy=access_policy)
|
|
787
|
-
if access_policy_copied
|
|
788
|
-
else FeedbackManager.prompt_s3_iamrole_connection_policy_not_copied(access_policy=access_policy)
|
|
789
|
-
),
|
|
790
|
-
show_default=False,
|
|
791
|
-
prompt_suffix="Press y to continue:",
|
|
792
|
-
):
|
|
793
|
-
sys.exit(1)
|
|
794
|
-
|
|
795
|
-
trust_policy_copied = True
|
|
796
|
-
try:
|
|
797
|
-
pyperclip.copy(trust_policy)
|
|
798
|
-
except Exception:
|
|
799
|
-
trust_policy_copied = False
|
|
800
|
-
|
|
801
|
-
if not click.confirm(
|
|
802
|
-
(
|
|
803
|
-
FeedbackManager.prompt_s3_iamrole_connection_role(trust_policy=trust_policy)
|
|
804
|
-
if trust_policy_copied
|
|
805
|
-
else FeedbackManager.prompt_s3_iamrole_connection_role_not_copied(trust_policy=trust_policy)
|
|
806
|
-
),
|
|
807
|
-
show_default=False,
|
|
808
|
-
prompt_suffix="Press y to continue:",
|
|
809
|
-
):
|
|
810
|
-
sys.exit(1)
|
|
811
|
-
else:
|
|
812
|
-
try:
|
|
813
|
-
trust_policy = await client.get_s3_trust_policy()
|
|
814
|
-
external_id = trust_policy["Statement"][0]["Condition"]["StringEquals"]["sts:ExternalId"]
|
|
815
|
-
except Exception:
|
|
816
|
-
external_id = ""
|
|
817
|
-
if not role_arn:
|
|
818
|
-
role_arn = click.prompt("Enter the ARN of the role you just created")
|
|
819
|
-
validate_string_connector_param("Role ARN", role_arn)
|
|
820
|
-
|
|
821
|
-
if not region:
|
|
822
|
-
region = click.prompt("Enter the region where the bucket is located")
|
|
823
|
-
validate_string_connector_param("Region", region)
|
|
824
|
-
|
|
825
|
-
return role_arn, region, external_id
|
|
826
|
-
|
|
827
|
-
async def validate_connection_name(connection_name: Optional[str]) -> str:
|
|
828
|
-
if connection_name and no_validate is False:
|
|
829
|
-
if await client.get_connector(connection_name) is not None:
|
|
830
|
-
raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
831
|
-
else:
|
|
832
|
-
while not connection_name:
|
|
833
|
-
connection_name = click.prompt("Enter the name for this connection", default=None, show_default=False)
|
|
834
|
-
assert isinstance(connection_name, str)
|
|
835
|
-
|
|
836
|
-
if no_validate is False:
|
|
837
|
-
if await client.get_connector(connection_name) is not None:
|
|
838
|
-
click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
839
|
-
connection_name = None
|
|
840
|
-
assert isinstance(connection_name, str)
|
|
841
|
-
return connection_name
|
|
842
|
-
|
|
843
|
-
async def create_connection(connection_name, role_arn, region) -> None:
|
|
844
|
-
conn_file_name = f"{connection_name}.connection"
|
|
845
|
-
conn_file_path = Path(getcwd(), conn_file_name)
|
|
846
|
-
service = DataConnectorType.AMAZON_S3_IAMROLE
|
|
847
|
-
|
|
848
|
-
if os.path.isfile(conn_file_path):
|
|
849
|
-
raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))
|
|
850
|
-
|
|
851
|
-
click.echo(FeedbackManager.info_creating_s3_iamrole_connection(connection_name=connection_name))
|
|
852
|
-
|
|
853
|
-
params = ConnectionReplacements.map_api_params_from_prompt_params(
|
|
854
|
-
service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
855
|
-
)
|
|
856
|
-
|
|
857
|
-
click.echo("** Creating connection...")
|
|
858
|
-
try:
|
|
859
|
-
_ = await client.connection_create(params)
|
|
860
|
-
except Exception as e:
|
|
861
|
-
raise CLIConnectionException(
|
|
862
|
-
FeedbackManager.error_connection_create(connection_name=connection_name, error=str(e))
|
|
863
|
-
)
|
|
864
|
-
|
|
865
|
-
with open(conn_file_path, "w") as f:
|
|
866
|
-
f.write(
|
|
867
|
-
f"""TYPE {service}
|
|
868
|
-
|
|
869
|
-
"""
|
|
870
|
-
)
|
|
871
|
-
click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
|
|
872
|
-
|
|
873
|
-
role_arn, region, external_id = await validate_s3_integration(role_arn, region)
|
|
874
|
-
connection_name = await validate_connection_name(connection_name)
|
|
875
|
-
await create_connection(connection_name, role_arn, region)
|
|
728
|
+
service = DataConnectorType.AMAZON_S3_IAMROLE
|
|
729
|
+
role_arn, region, external_id = await validate_aws_iamrole_integration(
|
|
730
|
+
client,
|
|
731
|
+
service=service,
|
|
732
|
+
role_arn=role_arn,
|
|
733
|
+
region=region,
|
|
734
|
+
policy=policy,
|
|
735
|
+
no_validate=no_validate,
|
|
736
|
+
)
|
|
737
|
+
connection_name = await validate_aws_iamrole_connection_name(client, connection_name, no_validate)
|
|
738
|
+
await create_aws_iamrole_connection(
|
|
739
|
+
client, service=service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
740
|
+
)
|
|
876
741
|
if external_id:
|
|
877
742
|
click.echo(
|
|
878
743
|
FeedbackManager.success_s3_iam_connection_created(
|
|
879
744
|
connection_name=connection_name, external_id=external_id, role_arn=role_arn
|
|
880
745
|
)
|
|
881
746
|
)
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
@connection_create.command(
|
|
750
|
+
name="dynamodb", short_help="Creates a AWS DynamoDB connection using IAM role authentication", hidden=True
|
|
751
|
+
)
|
|
752
|
+
@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
|
|
753
|
+
@click.option("--role-arn", default=None, help="The ARN of the IAM role to use for the connection")
|
|
754
|
+
@click.option("--region", default=None, help="The AWS region where DynamoDB is located")
|
|
755
|
+
@click.option("--no-validate", is_flag=True, default=False, help="Do not validate DynamoDB connection during creation")
|
|
756
|
+
@click.pass_context
|
|
757
|
+
@coro
|
|
758
|
+
async def connection_create_dynamodb(
|
|
759
|
+
ctx: Context,
|
|
760
|
+
connection_name: Optional[str] = "",
|
|
761
|
+
role_arn: Optional[str] = "",
|
|
762
|
+
region: Optional[str] = "",
|
|
763
|
+
no_validate: Optional[bool] = False,
|
|
764
|
+
) -> None:
|
|
765
|
+
"""
|
|
766
|
+
Creates a DynamoDB connection using IAM role authentication in the current workspace
|
|
767
|
+
|
|
768
|
+
\b
|
|
769
|
+
$ tb connection create dynamodb
|
|
770
|
+
"""
|
|
771
|
+
|
|
772
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
773
|
+
client: TinyB = obj["client"]
|
|
774
|
+
|
|
775
|
+
service = DataConnectorType.AMAZON_DYNAMODB
|
|
776
|
+
role_arn, region, _external_id = await validate_aws_iamrole_integration(
|
|
777
|
+
client,
|
|
778
|
+
service=service,
|
|
779
|
+
role_arn=role_arn,
|
|
780
|
+
region=region,
|
|
781
|
+
policy="read",
|
|
782
|
+
no_validate=no_validate,
|
|
783
|
+
)
|
|
784
|
+
connection_name = await validate_aws_iamrole_connection_name(client, connection_name, no_validate)
|
|
785
|
+
await create_aws_iamrole_connection(
|
|
786
|
+
client, service=service, connection_name=connection_name, role_arn=role_arn, region=region
|
|
787
|
+
)
|
|
788
|
+
click.echo(
|
|
789
|
+
FeedbackManager.success_dynamodb_connection_created(
|
|
790
|
+
connection_name=connection_name, region=region, role_arn=role_arn
|
|
791
|
+
)
|
|
792
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: tinybird-cli
|
|
3
|
-
Version: 5.2.2.
|
|
3
|
+
Version: 5.2.2.dev3
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli/introduction.html
|
|
6
6
|
Author: Tinybird
|
|
@@ -18,12 +18,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
18
18
|
Changelog
|
|
19
19
|
----------
|
|
20
20
|
|
|
21
|
+
|
|
22
|
+
5.2.2.dev3
|
|
23
|
+
**********
|
|
24
|
+
|
|
25
|
+
- `Added` `tb connection create` now supports `dynamodb` as service type
|
|
26
|
+
|
|
21
27
|
5.2.2.dev2
|
|
22
28
|
**********
|
|
29
|
+
|
|
23
30
|
- `Added` error when trying to push a data source with `SETTINGS` instead of `ENGINE_SETTINGS`
|
|
24
31
|
|
|
25
32
|
5.2.2.dev1
|
|
26
33
|
**********
|
|
34
|
+
|
|
27
35
|
- `Added` support for buckets with gzip files when creating S3 Data Sources
|
|
28
36
|
|
|
29
37
|
5.2.1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit.py
RENAMED
|
File without changes
|
{tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird/tb_cli_modules/workspace_members.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{tinybird-cli-5.2.2.dev2 → tinybird-cli-5.2.2.dev3}/tinybird_cli.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|