tinybird 4.5.12.dev0__tar.gz → 4.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/PKG-INFO +6 -2
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/datafile/common.py +17 -1
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/service_datasources.py +4 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/sql.py +8 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/__cli__.py +2 -2
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/client.py +20 -1
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/build_common.py +25 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/common.py +60 -26
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/connection.py +46 -0
- tinybird-4.6.0/tinybird/tb/modules/connection_dynamodb.py +240 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/connection_s3.py +2 -2
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/create.py +12 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build_pipe.py +2 -2
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datasource.py +166 -1
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/local_common.py +2 -2
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/project.py +10 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/secret.py +15 -1
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/test_common.py +1 -1
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird.egg-info/PKG-INFO +6 -2
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird.egg-info/SOURCES.txt +1 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/setup.cfg +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/__cli__.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/check_pypi.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/client.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/config.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/context.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/datafile/exceptions.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/datafile/parse_connection.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/datafile/parse_datasource.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/datafile/parse_pipe.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/datatypes.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/feedback_manager.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/git_settings.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/prompts.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/sql_template.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/sql_toolset.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/syncasync.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/check_pypi.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/cli.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/config.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/branch.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/build.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/cicd.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/cli.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/config.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/connection_kafka.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/copy.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build_common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/diff.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/fixture.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_connection.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/playground.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/pull.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/deployment.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/deployment_common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/deprecations.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/endpoint.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/exceptions.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/feedback_manager.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/fmt.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/info.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/infra.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/job.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/job_common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/llm.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/llm_utils.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/local.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/local_logs.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/login.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/login_common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/logout.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/logs.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/materialization.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/open.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/pipe.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/preview.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/project_commands.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/py_project.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/query_output.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/regions.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/secret_common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/sink.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/table.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/telemetry.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/test.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/token.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/ts_project.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/watch.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/workspace.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb/modules/workspace_members.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/connection.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird/tornado_template.py +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird.egg-info/dependency_links.txt +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird.egg-info/entry_points.txt +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird.egg-info/requires.txt +0 -0
- {tinybird-4.5.12.dev0 → tinybird-4.6.0}/tinybird.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.6.0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/forward/commands
|
|
6
6
|
Author: Tinybird
|
|
@@ -52,12 +52,16 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
52
52
|
Changelog
|
|
53
53
|
----------
|
|
54
54
|
|
|
55
|
+
4.6.0
|
|
56
|
+
*******
|
|
57
|
+
|
|
58
|
+
- `Added` Support for AWS DynamoDB datasource creation and management.
|
|
59
|
+
|
|
55
60
|
4.5.12
|
|
56
61
|
*******
|
|
57
62
|
|
|
58
63
|
- `Fixed` `tb info` now lists branches and marks the current branch correctly when running on a branch.
|
|
59
64
|
|
|
60
|
-
|
|
61
65
|
4.5.11
|
|
62
66
|
*******
|
|
63
67
|
|
|
@@ -277,6 +277,13 @@ VALID_BLOB_STORAGE_CRON_VALUES = {
|
|
|
277
277
|
"@auto",
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
+
REQUIRED_DYNAMODB_PARAMS = {
|
|
281
|
+
"import_connection_name",
|
|
282
|
+
"import_table_arn",
|
|
283
|
+
"import_export_bucket",
|
|
284
|
+
}
|
|
285
|
+
DYNAMODB_PARAMS = REQUIRED_DYNAMODB_PARAMS
|
|
286
|
+
|
|
280
287
|
|
|
281
288
|
def extract_column_names_from_sorting_key_part(part: str) -> List[str]:
|
|
282
289
|
"""
|
|
@@ -575,8 +582,15 @@ class Datafile:
|
|
|
575
582
|
)
|
|
576
583
|
if "kafka_group_id" in node and not str(node["kafka_group_id"]).strip():
|
|
577
584
|
raise DatafileValidationError("KAFKA_GROUP_ID cannot be empty")
|
|
585
|
+
# Validate DynamoDB params first since import_connection_name is shared with blob storage.
|
|
586
|
+
is_dynamodb = any(param in node for param in ("import_table_arn", "import_export_bucket"))
|
|
587
|
+
if is_dynamodb:
|
|
588
|
+
if missing := [param for param in REQUIRED_DYNAMODB_PARAMS if param not in node]:
|
|
589
|
+
raise DatafileValidationError(
|
|
590
|
+
f"Some DynamoDB params have been provided, but the following required ones are missing: {missing}"
|
|
591
|
+
)
|
|
578
592
|
# Validate S3 params
|
|
579
|
-
|
|
593
|
+
elif any(param in node for param in BLOB_STORAGE_PARAMS):
|
|
580
594
|
if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
|
|
581
595
|
raise DatafileValidationError(
|
|
582
596
|
f"Some connection params have been provided, but the following required ones are missing: {missing}"
|
|
@@ -2097,6 +2111,8 @@ def parse(
|
|
|
2097
2111
|
"s3_arn": assign_var("s3_arn"),
|
|
2098
2112
|
"s3_access_key": assign_var("s3_access_key"),
|
|
2099
2113
|
"s3_secret": assign_var("s3_secret"),
|
|
2114
|
+
"dynamodb_arn": assign_var("dynamodb_arn"),
|
|
2115
|
+
"dynamodb_region": assign_var("dynamodb_region"),
|
|
2100
2116
|
"gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
|
|
2101
2117
|
"gcs_access_id": assign_var("gcs_hmac_access_id"),
|
|
2102
2118
|
"gcs_secret": assign_var("gcs_hmac_secret"),
|
|
@@ -219,7 +219,9 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
|
|
|
219
219
|
{"name": "processed_messages", "type": "Int32"},
|
|
220
220
|
{"name": "processed_bytes", "type": "Int32"},
|
|
221
221
|
{"name": "committed_messages", "type": "Int32"},
|
|
222
|
+
{"name": "time_read", "type": "Float32"},
|
|
222
223
|
{"name": "time_process", "type": "Float32"},
|
|
224
|
+
{"name": "time_write", "type": "Float32"},
|
|
223
225
|
{"name": "msg", "type": "String"},
|
|
224
226
|
],
|
|
225
227
|
},
|
|
@@ -880,7 +882,9 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
|
|
|
880
882
|
{"name": "processed_messages", "type": "Int32"},
|
|
881
883
|
{"name": "processed_bytes", "type": "Int32"},
|
|
882
884
|
{"name": "committed_messages", "type": "Int32"},
|
|
885
|
+
{"name": "time_read", "type": "Float32"},
|
|
883
886
|
{"name": "time_process", "type": "Float32"},
|
|
887
|
+
{"name": "time_write", "type": "Float32"},
|
|
884
888
|
{"name": "msg", "type": "String"},
|
|
885
889
|
],
|
|
886
890
|
},
|
|
@@ -398,6 +398,14 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
|
|
|
398
398
|
>>> parse_table_structure('c Nullable(Float32) DEFAULT NULL')
|
|
399
399
|
[{'name': 'c', 'type': 'Float32', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': True, 'normalized_name': 'c'}]
|
|
400
400
|
|
|
401
|
+
>>> parse_table_structure('c LowCardinality(Nullable(String))')
|
|
402
|
+
[{'name': 'c', 'type': 'LowCardinality(Nullable(String))', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
|
|
403
|
+
|
|
404
|
+
>>> parse_table_structure('c Nullable(LowCardinality(Nullable(String)))')
|
|
405
|
+
Traceback (most recent call last):
|
|
406
|
+
...
|
|
407
|
+
ValueError: Nested type LowCardinality(Nullable(String)) cannot be inside Nullable type
|
|
408
|
+
|
|
401
409
|
>>> parse_table_structure("c String DEFAULT 'bla'")
|
|
402
410
|
[{'name': 'c', 'type': 'String', 'codec': None, 'default_value': "DEFAULT 'bla'", 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
|
|
403
411
|
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '4.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '4.6.0'
|
|
8
|
+
__revision__ = '1d9f802'
|
|
@@ -1183,6 +1183,21 @@ class TinyB:
|
|
|
1183
1183
|
except Exception:
|
|
1184
1184
|
return False
|
|
1185
1185
|
|
|
1186
|
+
def validate_dynamodb(
|
|
1187
|
+
self, table_arn: str, region: str, role_arn: str, external_id_seed: Optional[str] = None
|
|
1188
|
+
) -> Dict[str, Any]:
|
|
1189
|
+
body: Dict[str, Any] = {"table_arn": table_arn, "region": region, "role_arn": role_arn}
|
|
1190
|
+
if external_id_seed:
|
|
1191
|
+
body["external_id_seed"] = external_id_seed
|
|
1192
|
+
response = self._req_raw(
|
|
1193
|
+
"/v1/integrations/dynamodb/validate",
|
|
1194
|
+
method="POST",
|
|
1195
|
+
data=json.dumps(body),
|
|
1196
|
+
)
|
|
1197
|
+
if response.status_code >= 400:
|
|
1198
|
+
raise requests.HTTPError(parse_error_response(response), response=response)
|
|
1199
|
+
return response.json()
|
|
1200
|
+
|
|
1186
1201
|
def get_trust_policy(self, service: str, external_id_seed: Optional[str] = None) -> Dict[str, Any]:
|
|
1187
1202
|
params = {}
|
|
1188
1203
|
if external_id_seed:
|
|
@@ -1195,11 +1210,15 @@ class TinyB:
|
|
|
1195
1210
|
params["bucket"] = bucket
|
|
1196
1211
|
return self._req(f"/v0/integrations/{service}/policies/write-access-policy?{urlencode(params)}")
|
|
1197
1212
|
|
|
1198
|
-
def get_access_read_policy(
|
|
1213
|
+
def get_access_read_policy(
|
|
1214
|
+
self, service: str, bucket: Optional[str] = None, table_name: Optional[str] = None
|
|
1215
|
+
) -> Dict[str, Any]:
|
|
1199
1216
|
params = {}
|
|
1200
1217
|
if bucket:
|
|
1201
1218
|
# The Kafka endpoint scopes the policy via `msk_cluster_arn`, not `bucket`.
|
|
1202
1219
|
params["msk_cluster_arn" if service == "kafka" else "bucket"] = bucket
|
|
1220
|
+
if table_name:
|
|
1221
|
+
params["table_name"] = table_name
|
|
1203
1222
|
return self._req(f"/v0/integrations/{service}/policies/read-access-policy?{urlencode(params)}")
|
|
1204
1223
|
|
|
1205
1224
|
def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
|
@@ -247,6 +247,8 @@ def build_project(
|
|
|
247
247
|
if load_fixtures:
|
|
248
248
|
append_project_fixtures(project, tb_client, project_files)
|
|
249
249
|
echo_build_feedback(build.get("feedback", []))
|
|
250
|
+
if with_connections:
|
|
251
|
+
echo_dynamodb_local_backfill_feedback(build)
|
|
250
252
|
return True
|
|
251
253
|
elif build_result == "failed":
|
|
252
254
|
error = format_build_errors(result.get("errors", []))
|
|
@@ -461,6 +463,29 @@ def echo_build_feedback(feedback: list[dict[str, Any]]) -> None:
|
|
|
461
463
|
)
|
|
462
464
|
|
|
463
465
|
|
|
466
|
+
def echo_dynamodb_local_backfill_feedback(build: dict[str, Any]) -> None:
|
|
467
|
+
datasources_by_id = {datasource.get("id"): datasource.get("name") for datasource in build.get("datasources", [])}
|
|
468
|
+
for data_linker in build.get("data_linkers", []):
|
|
469
|
+
if data_linker.get("service") != "dynamodb":
|
|
470
|
+
continue
|
|
471
|
+
|
|
472
|
+
settings = data_linker.get("settings") or {}
|
|
473
|
+
export_arn = settings.get("initial_export_arn")
|
|
474
|
+
if not export_arn:
|
|
475
|
+
continue
|
|
476
|
+
|
|
477
|
+
datasource_name = datasources_by_id.get(data_linker.get("datasource_id")) or "unknown"
|
|
478
|
+
click.echo(
|
|
479
|
+
FeedbackManager.warning(
|
|
480
|
+
message=(
|
|
481
|
+
f"△ DynamoDB initial export backfill started for datasource '{datasource_name}'. "
|
|
482
|
+
"AWS exports can stay in progress for several minutes; Tinybird Local will keep retrying "
|
|
483
|
+
f"the import until AWS marks the export as completed. Export ARN: {export_arn}."
|
|
484
|
+
)
|
|
485
|
+
)
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
464
489
|
def format_build_errors(build_errors: list[dict[str, Any]]) -> str:
|
|
465
490
|
full_error_msg = ""
|
|
466
491
|
for build_error in build_errors:
|
|
@@ -1623,7 +1623,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1623
1623
|
local_unavailable: bool = False,
|
|
1624
1624
|
) -> Tuple[str, str, Optional[TinyB], Optional[TinyB]]:
|
|
1625
1625
|
"""
|
|
1626
|
-
Run the interactive AWS IAM Role connection flow for S3.
|
|
1626
|
+
Run the interactive AWS IAM Role connection flow for S3 or DynamoDB.
|
|
1627
1627
|
|
|
1628
1628
|
Guides the user through creating an IAM policy and role with the appropriate
|
|
1629
1629
|
trust policy that includes AWS account IDs from the selected environments.
|
|
@@ -1631,7 +1631,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1631
1631
|
Args:
|
|
1632
1632
|
config: The CLI configuration dictionary.
|
|
1633
1633
|
client: The TinyB client instance.
|
|
1634
|
-
service: The data connector service type (e.g., 's3').
|
|
1634
|
+
service: The data connector service type (e.g., 's3' or 'dynamodb').
|
|
1635
1635
|
connection_name: The name for the connection being created.
|
|
1636
1636
|
policy: The access policy type ('read' or 'write').
|
|
1637
1637
|
local_unavailable: If True, local environment is unavailable (e.g., missing AWS credentials).
|
|
@@ -1639,23 +1639,41 @@ def run_aws_iamrole_connection_flow(
|
|
|
1639
1639
|
Returns:
|
|
1640
1640
|
A tuple containing:
|
|
1641
1641
|
- role_arn (str): The AWS IAM Role ARN entered by the user.
|
|
1642
|
-
- region (str): The AWS region where the
|
|
1642
|
+
- region (str): The AWS region where the resource is located.
|
|
1643
1643
|
- cloud_client (Optional[TinyB]): The TinyB client instance for the cloud environment.
|
|
1644
1644
|
- local_client (Optional[TinyB]): The TinyB client instance for the local environment.
|
|
1645
1645
|
"""
|
|
1646
|
+
bucket_name: Optional[str]
|
|
1647
|
+
table_name: Optional[str] = None
|
|
1648
|
+
region_prompt: str
|
|
1646
1649
|
if service == DataConnectorType.AMAZON_DYNAMODB:
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
)
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1650
|
+
table_name = click.prompt(
|
|
1651
|
+
FeedbackManager.highlight(
|
|
1652
|
+
message="? DynamoDB table name (specific name recommended, use '*' for unrestricted access in IAM policy)"
|
|
1653
|
+
),
|
|
1654
|
+
prompt_suffix="\n> ",
|
|
1655
|
+
)
|
|
1656
|
+
validate_string_connector_param("DynamoDB table name", table_name)
|
|
1657
|
+
bucket_name = click.prompt(
|
|
1658
|
+
FeedbackManager.highlight(
|
|
1659
|
+
message="? Export bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
|
|
1660
|
+
),
|
|
1661
|
+
prompt_suffix="\n> ",
|
|
1662
|
+
)
|
|
1663
|
+
validate_string_connector_param("Export bucket name", bucket_name)
|
|
1664
|
+
region_prompt = "? Region (the region where the DynamoDB table is located)"
|
|
1665
|
+
else:
|
|
1666
|
+
bucket_name = click.prompt(
|
|
1667
|
+
FeedbackManager.highlight(
|
|
1668
|
+
message="? Bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
|
|
1669
|
+
),
|
|
1670
|
+
prompt_suffix="\n> ",
|
|
1671
|
+
)
|
|
1672
|
+
validate_string_connector_param("Bucket", bucket_name)
|
|
1673
|
+
region_prompt = "? Region (the region where the bucket is located)"
|
|
1656
1674
|
|
|
1657
1675
|
region = click.prompt(
|
|
1658
|
-
FeedbackManager.highlight(message=
|
|
1676
|
+
FeedbackManager.highlight(message=region_prompt),
|
|
1659
1677
|
default="us-east-1",
|
|
1660
1678
|
show_default=True,
|
|
1661
1679
|
prompt_suffix="\n> ",
|
|
@@ -1738,15 +1756,27 @@ def run_aws_iamrole_connection_flow(
|
|
|
1738
1756
|
# Use cloud_client as the main client if local is unavailable
|
|
1739
1757
|
policy_client = cloud_client if local_unavailable and cloud_client else client
|
|
1740
1758
|
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1759
|
+
if table_name is not None:
|
|
1760
|
+
access_policy, trust_policy, _ = get_aws_iamrole_policies(
|
|
1761
|
+
policy_client,
|
|
1762
|
+
service=service,
|
|
1763
|
+
policy=policy,
|
|
1764
|
+
bucket=bucket_name,
|
|
1765
|
+
table_name=table_name,
|
|
1766
|
+
external_id_seed=connection_name,
|
|
1767
|
+
cloud_client=cloud_client if not local_unavailable else None,
|
|
1768
|
+
local_client=local_client,
|
|
1769
|
+
)
|
|
1770
|
+
else:
|
|
1771
|
+
access_policy, trust_policy, _ = get_aws_iamrole_policies(
|
|
1772
|
+
policy_client,
|
|
1773
|
+
service=service,
|
|
1774
|
+
policy=policy,
|
|
1775
|
+
bucket=bucket_name,
|
|
1776
|
+
external_id_seed=connection_name,
|
|
1777
|
+
cloud_client=cloud_client if not local_unavailable else None,
|
|
1778
|
+
local_client=local_client,
|
|
1779
|
+
)
|
|
1750
1780
|
|
|
1751
1781
|
click.echo(FeedbackManager.gray(message="\n» Step 1: AWS Authentication"))
|
|
1752
1782
|
click.echo(
|
|
@@ -1756,7 +1786,10 @@ def run_aws_iamrole_connection_flow(
|
|
|
1756
1786
|
)
|
|
1757
1787
|
click.echo(
|
|
1758
1788
|
FeedbackManager.info(
|
|
1759
|
-
message=
|
|
1789
|
+
message=(
|
|
1790
|
+
"You'll be creating a single IAM Policy and Role to access your data. "
|
|
1791
|
+
"Using IAM Roles improves security by providing temporary credentials and following least privilege principles."
|
|
1792
|
+
)
|
|
1760
1793
|
)
|
|
1761
1794
|
)
|
|
1762
1795
|
click.echo(FeedbackManager.click_enter_to_continue())
|
|
@@ -1781,7 +1814,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1781
1814
|
click.echo(FeedbackManager.info(message="3. Copy and paste the following policy:"))
|
|
1782
1815
|
click.echo(FeedbackManager.highlight(message=f"\n{access_policy}\n"))
|
|
1783
1816
|
click.echo(
|
|
1784
|
-
FeedbackManager.info(message=f"4. Name the policy something meaningful (e.g.,
|
|
1817
|
+
FeedbackManager.info(message=f"4. Name the policy something meaningful (e.g., TinybirdAccess-{bucket_name})")
|
|
1785
1818
|
)
|
|
1786
1819
|
click.echo(FeedbackManager.info(message="5. Click 'Create policy'"))
|
|
1787
1820
|
click.echo(FeedbackManager.click_enter_to_continue())
|
|
@@ -1807,7 +1840,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1807
1840
|
click.echo(FeedbackManager.highlight(message=f"\n{trust_policy}\n"))
|
|
1808
1841
|
click.echo(FeedbackManager.info(message="4. Click Next, search for and select the policy you just created"))
|
|
1809
1842
|
click.echo(
|
|
1810
|
-
FeedbackManager.info(message=f"5. Name the role something meaningful (e.g.,
|
|
1843
|
+
FeedbackManager.info(message=f"5. Name the role something meaningful (e.g., TinybirdRole-{bucket_name})")
|
|
1811
1844
|
)
|
|
1812
1845
|
click.echo(FeedbackManager.info(message="6. Click 'Create role'"))
|
|
1813
1846
|
click.echo(FeedbackManager.info(message="7. Copy the Role ARN from the role details page"))
|
|
@@ -1954,6 +1987,7 @@ def get_aws_iamrole_policies(
|
|
|
1954
1987
|
service: str,
|
|
1955
1988
|
policy: str = "write",
|
|
1956
1989
|
bucket: Optional[str] = None,
|
|
1990
|
+
table_name: Optional[str] = None,
|
|
1957
1991
|
external_id_seed: Optional[str] = None,
|
|
1958
1992
|
cloud_client: Optional[TinyB] = None,
|
|
1959
1993
|
local_client: Optional[TinyB] = None,
|
|
@@ -1979,7 +2013,7 @@ def get_aws_iamrole_policies(
|
|
|
1979
2013
|
if policy == "write":
|
|
1980
2014
|
access_policy = client.get_access_write_policy(service, bucket)
|
|
1981
2015
|
elif policy == "read":
|
|
1982
|
-
access_policy = client.get_access_read_policy(service, bucket)
|
|
2016
|
+
access_policy = client.get_access_read_policy(service, bucket, table_name=table_name)
|
|
1983
2017
|
else:
|
|
1984
2018
|
raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
|
|
1985
2019
|
if not len(access_policy) > 0:
|
|
@@ -20,6 +20,7 @@ from tinybird.tb.modules.common import (
|
|
|
20
20
|
get_gcs_svc_account_creds,
|
|
21
21
|
run_gcp_svc_account_connection_flow,
|
|
22
22
|
)
|
|
23
|
+
from tinybird.tb.modules.connection_dynamodb import connection_create_dynamodb
|
|
23
24
|
from tinybird.tb.modules.connection_kafka import (
|
|
24
25
|
connection_create_kafka,
|
|
25
26
|
echo_kafka_data,
|
|
@@ -248,6 +249,51 @@ def connection_create_gcs(ctx: Context) -> None:
|
|
|
248
249
|
)
|
|
249
250
|
|
|
250
251
|
|
|
252
|
+
@connection_create.command(
|
|
253
|
+
name="dynamodb", short_help="Creates a AWS DynamoDB connection using IAM role authentication."
|
|
254
|
+
)
|
|
255
|
+
@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
|
|
256
|
+
@click.option("--table-arn", default=None, help="Optional. Validate the connection against this DynamoDB table ARN")
|
|
257
|
+
@click.pass_context
|
|
258
|
+
def connection_create_dynamodb_cmd(ctx: Context, connection_name: Optional[str], table_arn: Optional[str]) -> None:
|
|
259
|
+
"""
|
|
260
|
+
Creates a AWS DynamoDB connection using IAM role authentication.
|
|
261
|
+
|
|
262
|
+
\b
|
|
263
|
+
$ tb connection create dynamodb
|
|
264
|
+
"""
|
|
265
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
266
|
+
project: Project = obj["project"]
|
|
267
|
+
client: TinyB = obj["client"]
|
|
268
|
+
env: str = obj["env"]
|
|
269
|
+
config = obj["config"]
|
|
270
|
+
|
|
271
|
+
local_aws_unavailable = env == "local" and not client.check_aws_credentials()
|
|
272
|
+
|
|
273
|
+
if env == "local" and not local_aws_unavailable:
|
|
274
|
+
click.echo(FeedbackManager.gray(message="» Building project before continue..."))
|
|
275
|
+
error = build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
|
|
276
|
+
if error:
|
|
277
|
+
click.echo(FeedbackManager.error(message=error))
|
|
278
|
+
else:
|
|
279
|
+
click.echo(FeedbackManager.success(message="✓ Build completed"))
|
|
280
|
+
|
|
281
|
+
result = connection_create_dynamodb(ctx, connection_name=connection_name, table_arn=table_arn)
|
|
282
|
+
|
|
283
|
+
if env == "local" and not local_aws_unavailable and not result["error"]:
|
|
284
|
+
click.echo(FeedbackManager.gray(message="» Building project to access the new connection..."))
|
|
285
|
+
error = build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
|
|
286
|
+
if error:
|
|
287
|
+
click.echo(FeedbackManager.error(message=error))
|
|
288
|
+
else:
|
|
289
|
+
click.echo(FeedbackManager.success(message="✓ Build completed"))
|
|
290
|
+
|
|
291
|
+
# connection_create_dynamodb already prints the failure details; exit non-zero so scripted/CI
|
|
292
|
+
# usage does not treat a connection with missing secrets as a success.
|
|
293
|
+
if result["error"]:
|
|
294
|
+
ctx.exit(1)
|
|
295
|
+
|
|
296
|
+
|
|
251
297
|
@connection_create.command(name="kafka", help="Create a Kafka connection.")
|
|
252
298
|
@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
|
|
253
299
|
@click.option("--bootstrap-servers", default=None, help="Kafka Bootstrap Server in form mykafka.mycloud.com:9092")
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from typing import Any, Optional
|
|
3
|
+
|
|
4
|
+
import click
|
|
5
|
+
import requests
|
|
6
|
+
from click import Context
|
|
7
|
+
|
|
8
|
+
from tinybird.tb.client import TinyB
|
|
9
|
+
from tinybird.tb.modules.common import (
|
|
10
|
+
DataConnectorType,
|
|
11
|
+
get_connection_name,
|
|
12
|
+
run_aws_iamrole_connection_flow,
|
|
13
|
+
validate_string_connector_param,
|
|
14
|
+
)
|
|
15
|
+
from tinybird.tb.modules.create import generate_dynamodb_connection_file_with_secret
|
|
16
|
+
from tinybird.tb.modules.feedback_manager import FeedbackManager, get_cli_name
|
|
17
|
+
from tinybird.tb.modules.project import Project
|
|
18
|
+
from tinybird.tb.modules.secret import save_secret_to_local_environment
|
|
19
|
+
|
|
20
|
+
_DYNAMODB_VALIDATE_REASON_MESSAGES: dict[str, str] = {
|
|
21
|
+
"missing_credentials": (
|
|
22
|
+
"Tinybird could not validate the DynamoDB table because AWS credentials are missing in this environment."
|
|
23
|
+
),
|
|
24
|
+
"table_not_found": "The DynamoDB table was not found. Check the table ARN and region.",
|
|
25
|
+
"pitr_disabled": "Point-in-Time Recovery (PITR) must be enabled to use the DynamoDB connector.",
|
|
26
|
+
"stream_disabled": "DynamoDB Streams must be enabled to use the DynamoDB connector.",
|
|
27
|
+
"stream_view_invalid": "DynamoDB Streams must use NEW_IMAGE or NEW_AND_OLD_IMAGES.",
|
|
28
|
+
"table_too_large": "The DynamoDB table exceeds the current size limit for this connector.",
|
|
29
|
+
"table_wcu_exceeds_limit": "The DynamoDB table exceeds the current write-capacity limit for this connector.",
|
|
30
|
+
"table_arn_and_region_required": "Both the DynamoDB table ARN and region are required for validation.",
|
|
31
|
+
"role_arn_required": "A role ARN is required to validate the DynamoDB table.",
|
|
32
|
+
"unable_to_assume_role": (
|
|
33
|
+
"Tinybird could not assume the provided IAM role. Check the role ARN and its trust policy."
|
|
34
|
+
),
|
|
35
|
+
"invalid_json_body": "Tinybird returned an invalid validation request error.",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _extract_reason_from_validation_error(exc: Exception) -> Optional[str]:
|
|
40
|
+
response = getattr(exc, "response", None)
|
|
41
|
+
if response is not None:
|
|
42
|
+
try:
|
|
43
|
+
payload = response.json()
|
|
44
|
+
except Exception:
|
|
45
|
+
pass
|
|
46
|
+
else:
|
|
47
|
+
error = payload.get("error")
|
|
48
|
+
if isinstance(error, dict):
|
|
49
|
+
return error.get("reason")
|
|
50
|
+
|
|
51
|
+
error_message = str(exc)
|
|
52
|
+
for reason in _DYNAMODB_VALIDATE_REASON_MESSAGES:
|
|
53
|
+
if reason in error_message:
|
|
54
|
+
return reason
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _format_dynamodb_validation_message(reason: Optional[str], fallback: str) -> str:
|
|
59
|
+
if reason and reason in _DYNAMODB_VALIDATE_REASON_MESSAGES:
|
|
60
|
+
return _DYNAMODB_VALIDATE_REASON_MESSAGES[reason]
|
|
61
|
+
return fallback
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def validate_dynamodb_table(
|
|
65
|
+
client: TinyB,
|
|
66
|
+
table_arn: str,
|
|
67
|
+
region: str,
|
|
68
|
+
role_arn: str,
|
|
69
|
+
*,
|
|
70
|
+
fail_on_error: bool,
|
|
71
|
+
external_id_seed: Optional[str] = None,
|
|
72
|
+
) -> Optional[dict[str, Any]]:
|
|
73
|
+
try:
|
|
74
|
+
result = client.validate_dynamodb(
|
|
75
|
+
table_arn=table_arn, region=region, role_arn=role_arn, external_id_seed=external_id_seed
|
|
76
|
+
)
|
|
77
|
+
except requests.HTTPError as exc:
|
|
78
|
+
message = _format_dynamodb_validation_message(_extract_reason_from_validation_error(exc), str(exc))
|
|
79
|
+
if fail_on_error:
|
|
80
|
+
raise click.ClickException(FeedbackManager.error(message=message))
|
|
81
|
+
click.echo(FeedbackManager.warning(message=message))
|
|
82
|
+
return None
|
|
83
|
+
except Exception as exc:
|
|
84
|
+
if fail_on_error:
|
|
85
|
+
raise click.ClickException(FeedbackManager.error(message=str(exc)))
|
|
86
|
+
click.echo(FeedbackManager.warning(message=f"DynamoDB validation failed: {exc}"))
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
click.echo(FeedbackManager.success(message="✓ DynamoDB validation passed"))
|
|
90
|
+
view_type = result.get("stream_view_type")
|
|
91
|
+
if view_type:
|
|
92
|
+
click.echo(FeedbackManager.gray(message=f" Streams view type: {view_type}"))
|
|
93
|
+
if result.get("table_size_bytes") is not None:
|
|
94
|
+
click.echo(FeedbackManager.gray(message=f" Table size bytes: {result['table_size_bytes']}"))
|
|
95
|
+
if result.get("table_write_capacity_units") is not None:
|
|
96
|
+
click.echo(
|
|
97
|
+
FeedbackManager.gray(message=f" Table write capacity units: {result['table_write_capacity_units']}")
|
|
98
|
+
)
|
|
99
|
+
for warning in result.get("messages", []):
|
|
100
|
+
click.echo(FeedbackManager.warning(message=str(warning)))
|
|
101
|
+
return result
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _read_optional_table_arn_for_validation(table_arn: Optional[str]) -> Optional[str]:
|
|
105
|
+
if table_arn is not None:
|
|
106
|
+
return table_arn.strip() or None
|
|
107
|
+
|
|
108
|
+
user_input = click.prompt(
|
|
109
|
+
FeedbackManager.highlight(message="? Optional DynamoDB table ARN to validate now (press Enter to skip)"),
|
|
110
|
+
default="",
|
|
111
|
+
show_default=False,
|
|
112
|
+
)
|
|
113
|
+
user_input = user_input.strip()
|
|
114
|
+
return user_input or None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def connection_create_dynamodb(
|
|
118
|
+
ctx: Context,
|
|
119
|
+
connection_name: Optional[str] = None,
|
|
120
|
+
table_arn: Optional[str] = None,
|
|
121
|
+
) -> dict[str, Any]:
|
|
122
|
+
obj: dict[str, Any] = ctx.ensure_object(dict)
|
|
123
|
+
project: Project = obj["project"]
|
|
124
|
+
client: TinyB = obj["client"]
|
|
125
|
+
env: str = obj["env"]
|
|
126
|
+
config: dict[str, Any] = obj["config"]
|
|
127
|
+
|
|
128
|
+
local_aws_unavailable = False
|
|
129
|
+
if env == "local" and not client.check_aws_credentials():
|
|
130
|
+
click.echo(
|
|
131
|
+
FeedbackManager.warning(
|
|
132
|
+
message=(
|
|
133
|
+
f"No AWS credentials found. Please run `{get_cli_name()} local restart --use-aws-creds` "
|
|
134
|
+
"to pass your credentials. Read more about this in "
|
|
135
|
+
"https://www.tinybird.co/docs/forward/get-data-in/connectors/dynamodb#local-environment"
|
|
136
|
+
)
|
|
137
|
+
)
|
|
138
|
+
)
|
|
139
|
+
click.echo(
|
|
140
|
+
FeedbackManager.warning(
|
|
141
|
+
message=(
|
|
142
|
+
"Continuing without Tinybird Local. Only Cloud environment will be available for this connection."
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
local_aws_unavailable = True
|
|
147
|
+
|
|
148
|
+
click.echo(FeedbackManager.gray(message="\n» Creating DynamoDB connection..."))
|
|
149
|
+
|
|
150
|
+
if not connection_name:
|
|
151
|
+
connection_name = get_connection_name(project.folder, "DYNAMODB")
|
|
152
|
+
validate_string_connector_param("Connection name", connection_name)
|
|
153
|
+
|
|
154
|
+
role_arn, region, cloud_client, local_client = run_aws_iamrole_connection_flow(
|
|
155
|
+
config=config,
|
|
156
|
+
client=client,
|
|
157
|
+
service=DataConnectorType.AMAZON_DYNAMODB,
|
|
158
|
+
connection_name=connection_name,
|
|
159
|
+
policy="read",
|
|
160
|
+
local_unavailable=local_aws_unavailable,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
unique_suffix = uuid.uuid4().hex[:8]
|
|
164
|
+
secret_name = f"dynamodb_role_arn_{connection_name}_{unique_suffix}"
|
|
165
|
+
secret_created_local = False
|
|
166
|
+
secret_created_cloud = False
|
|
167
|
+
errors: list[str] = []
|
|
168
|
+
|
|
169
|
+
if local_client:
|
|
170
|
+
try:
|
|
171
|
+
save_secret_to_local_environment(project=project, name=secret_name, value=role_arn, client=local_client)
|
|
172
|
+
secret_created_local = True
|
|
173
|
+
except Exception as exc:
|
|
174
|
+
errors.append(f"Failed to create secret in local: {exc}")
|
|
175
|
+
click.echo(FeedbackManager.warning(message=f"Failed to create secret in local: {exc}"))
|
|
176
|
+
|
|
177
|
+
if cloud_client:
|
|
178
|
+
try:
|
|
179
|
+
cloud_client.create_secret(name=secret_name, value=role_arn)
|
|
180
|
+
secret_created_cloud = True
|
|
181
|
+
except Exception as exc:
|
|
182
|
+
errors.append(f"Failed to create secret in cloud: {exc}")
|
|
183
|
+
click.echo(FeedbackManager.warning(message=f"Failed to create secret in cloud: {exc}"))
|
|
184
|
+
|
|
185
|
+
connection_file_path = generate_dynamodb_connection_file_with_secret(
|
|
186
|
+
name=connection_name,
|
|
187
|
+
role_arn_secret_name=secret_name,
|
|
188
|
+
region=region,
|
|
189
|
+
folder=project.folder,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
validate_table_arn = _read_optional_table_arn_for_validation(table_arn)
|
|
193
|
+
if validate_table_arn:
|
|
194
|
+
click.echo(FeedbackManager.gray(message="\n» Validating DynamoDB table..."))
|
|
195
|
+
validate_dynamodb_table(
|
|
196
|
+
client,
|
|
197
|
+
validate_table_arn,
|
|
198
|
+
region,
|
|
199
|
+
role_arn,
|
|
200
|
+
fail_on_error=False,
|
|
201
|
+
external_id_seed=connection_name,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
items = [f"- File created at: {connection_file_path}"]
|
|
205
|
+
if secret_created_local and secret_created_cloud:
|
|
206
|
+
items.append(f"- Secret created in Local and Cloud for role ARN with name {secret_name}")
|
|
207
|
+
elif secret_created_local:
|
|
208
|
+
items.append(f"- Secret created in Local for role ARN with name {secret_name}")
|
|
209
|
+
elif secret_created_cloud:
|
|
210
|
+
items.append(f"- Secret created in Cloud for role ARN with name {secret_name}")
|
|
211
|
+
|
|
212
|
+
if errors:
|
|
213
|
+
click.echo(
|
|
214
|
+
FeedbackManager.error(
|
|
215
|
+
message=(
|
|
216
|
+
f"DynamoDB connection '{connection_name}' could not be created. "
|
|
217
|
+
f"Review the configuration at: {connection_file_path}"
|
|
218
|
+
)
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
for error in errors:
|
|
222
|
+
click.echo(FeedbackManager.error(message=f" - {error}"))
|
|
223
|
+
return {"name": connection_name, "error": "; ".join(errors)}
|
|
224
|
+
|
|
225
|
+
click.echo(
|
|
226
|
+
FeedbackManager.success(
|
|
227
|
+
message=f"DynamoDB connection '{connection_name}' created successfully!\n" + "\n".join(items)
|
|
228
|
+
)
|
|
229
|
+
)
|
|
230
|
+
click.echo(
|
|
231
|
+
FeedbackManager.gray(
|
|
232
|
+
message=(
|
|
233
|
+
f"Next steps:\n- Use this connection in your Data Sources with: "
|
|
234
|
+
f"IMPORT_CONNECTION_NAME '{connection_name}'\n"
|
|
235
|
+
"- Learn more about our DynamoDB Connector: "
|
|
236
|
+
"https://www.tinybird.co/docs/forward/get-data-in/connectors/dynamodb"
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
)
|
|
240
|
+
return {"name": connection_name, "error": None}
|
|
@@ -20,7 +20,7 @@ from tinybird.tb.modules.create import generate_aws_iamrole_connection_file_with
|
|
|
20
20
|
from tinybird.tb.modules.exceptions import CLIConnectionException
|
|
21
21
|
from tinybird.tb.modules.feedback_manager import FeedbackManager, get_cli_name
|
|
22
22
|
from tinybird.tb.modules.project import Project
|
|
23
|
-
from tinybird.tb.modules.secret import
|
|
23
|
+
from tinybird.tb.modules.secret import save_secret_to_local_environment
|
|
24
24
|
|
|
25
25
|
|
|
26
26
|
def select_bucket_uri(bucket_uri: Optional[str]) -> str:
|
|
@@ -375,7 +375,7 @@ def connection_create_s3(
|
|
|
375
375
|
# Create secrets only in selected environments
|
|
376
376
|
if local_client:
|
|
377
377
|
try:
|
|
378
|
-
|
|
378
|
+
save_secret_to_local_environment(project=project, name=secret_name, value=role_arn, client=local_client)
|
|
379
379
|
secret_created_local = True
|
|
380
380
|
except Exception as e:
|
|
381
381
|
errors.append(f"Failed to create secret in local: {e}")
|