tinybird 4.5.11.dev0__tar.gz → 4.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/PKG-INFO +18 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/datafile/common.py +17 -1
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/service_datasources.py +4 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/sql.py +8 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/__cli__.py +2 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/client.py +37 -4
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/branch.py +14 -5
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/build_common.py +25 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/cli.py +32 -3
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/common.py +107 -40
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/connection.py +69 -2
- tinybird-4.6.0/tinybird/tb/modules/connection_dynamodb.py +240 -0
- tinybird-4.6.0/tinybird/tb/modules/connection_kafka.py +804 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/connection_s3.py +2 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/create.py +46 -11
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build.py +32 -5
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build_pipe.py +2 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datasource.py +166 -1
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/feedback_manager.py +3 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/info.py +35 -11
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/local_common.py +2 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/preview.py +3 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/project.py +10 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/secret.py +15 -1
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/test_common.py +1 -1
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/connection.py +80 -9
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird.egg-info/PKG-INFO +18 -2
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird.egg-info/SOURCES.txt +1 -0
- tinybird-4.5.11.dev0/tinybird/tb/modules/connection_kafka.py +0 -433
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/setup.cfg +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/__cli__.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/ch_utils/constants.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/ch_utils/engine.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/check_pypi.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/client.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/config.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/context.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/datafile/exceptions.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/datafile/parse_connection.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/datafile/parse_datasource.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/datafile/parse_pipe.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/datatypes.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/feedback_manager.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/git_settings.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/prompts.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/sql_template.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/sql_template_fmt.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/sql_toolset.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/syncasync.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/check_pypi.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/cli.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/config.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/build.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/cicd.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/config.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/copy.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build_common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/build_datasource.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/diff.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/fixture.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_connection.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_datasource.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/format_pipe.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/pipe_checker.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/playground.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/datafile/pull.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/deployment.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/deployment_common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/deprecations.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/endpoint.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/exceptions.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/fmt.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/infra.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/job.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/job_common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/llm.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/llm_utils.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/local.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/local_logs.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/login.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/login_common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/logout.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/logs.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/materialization.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/open.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/pipe.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/project_commands.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/py_project.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/query_output.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/regions.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/secret_common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/sink.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/table.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/telemetry.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/test.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/tinyunit/tinyunit.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/token.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/ts_project.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/watch.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/workspace.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb/modules/workspace_members.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/auth.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/branch.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/cicd.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/cli.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/common.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/config.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/datasource.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/exceptions.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/fmt.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/job.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/pipe.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/regions.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/tag.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/telemetry.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/test.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/tinyunit/tinyunit.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/workspace.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tb_cli_modules/workspace_members.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird/tornado_template.py +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird.egg-info/dependency_links.txt +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird.egg-info/entry_points.txt +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird.egg-info/requires.txt +0 -0
- {tinybird-4.5.11.dev0 → tinybird-4.6.0}/tinybird.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.6.0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/forward/commands
|
|
6
6
|
Author: Tinybird
|
|
@@ -52,10 +52,26 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
52
52
|
Changelog
|
|
53
53
|
----------
|
|
54
54
|
|
|
55
|
-
4.
|
|
55
|
+
4.6.0
|
|
56
56
|
*******
|
|
57
57
|
|
|
58
|
+
- `Added` Support for AWS DynamoDB datasource creation and management.
|
|
59
|
+
|
|
60
|
+
4.5.12
|
|
61
|
+
*******
|
|
62
|
+
|
|
63
|
+
- `Fixed` `tb info` now lists branches and marks the current branch correctly when running on a branch.
|
|
64
|
+
|
|
65
|
+
4.5.11
|
|
66
|
+
*******
|
|
67
|
+
|
|
68
|
+
- `Added` `tb connection create kafka` now supports SASL ``OAUTHBEARER`` for Amazon MSK clusters with IAM-based authentication. The wizard asks up-front whether the user already has an IAM role for the cluster — if yes, it collects region + role ARN + external ID directly; if no, it walks the user through creating the IAM access policy and trust policy in the AWS Console, copying them to the clipboard.
|
|
69
|
+
- `Added` ``--oauthbearer-aws-external-id`` flag (and matching interactive prompt) so users can supply a pre-shared external ID instead of the one Tinybird auto-generates for the connection.
|
|
70
|
+
- `Added` ``PLAINTEXT`` security protocol now skips the SASL mechanism prompt entirely.
|
|
71
|
+
- `Added` ``.connection`` files generated by the wizard now emit ``KAFKA_SASL_OAUTHBEARER_METHOD``, ``KAFKA_SASL_OAUTHBEARER_AWS_REGION``, ``KAFKA_SASL_OAUTHBEARER_AWS_ROLE_ARN``, and ``KAFKA_SASL_OAUTHBEARER_AWS_EXTERNAL_ID``, with the role ARN stored as a ``tb_secret``.
|
|
72
|
+
- `Fixed` ``tb connection create kafka`` now accepts comma-separated bootstrap-server lists (e.g. ``b-1.kafka:9092,b-2.kafka:9092``) instead of rejecting them as malformed. Validation passes as soon as any one broker is reachable, matching Kafka bootstrap semantics.
|
|
58
73
|
- `Fixed` `tb init` now persists `folder` in `tinybird.config.json`, so subsequent resource creation targets the configured project folder instead of the repository root.
|
|
74
|
+
- `Fixed` Branch names derived from Git refs are now sanitized in `tb preview --name` and `tb branch create/rm/clear`, preventing CI/CD failures with invalid workspace names.
|
|
59
75
|
|
|
60
76
|
4.5.9
|
|
61
77
|
*******
|
|
@@ -277,6 +277,13 @@ VALID_BLOB_STORAGE_CRON_VALUES = {
|
|
|
277
277
|
"@auto",
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
+
REQUIRED_DYNAMODB_PARAMS = {
|
|
281
|
+
"import_connection_name",
|
|
282
|
+
"import_table_arn",
|
|
283
|
+
"import_export_bucket",
|
|
284
|
+
}
|
|
285
|
+
DYNAMODB_PARAMS = REQUIRED_DYNAMODB_PARAMS
|
|
286
|
+
|
|
280
287
|
|
|
281
288
|
def extract_column_names_from_sorting_key_part(part: str) -> List[str]:
|
|
282
289
|
"""
|
|
@@ -575,8 +582,15 @@ class Datafile:
|
|
|
575
582
|
)
|
|
576
583
|
if "kafka_group_id" in node and not str(node["kafka_group_id"]).strip():
|
|
577
584
|
raise DatafileValidationError("KAFKA_GROUP_ID cannot be empty")
|
|
585
|
+
# Validate DynamoDB params first since import_connection_name is shared with blob storage.
|
|
586
|
+
is_dynamodb = any(param in node for param in ("import_table_arn", "import_export_bucket"))
|
|
587
|
+
if is_dynamodb:
|
|
588
|
+
if missing := [param for param in REQUIRED_DYNAMODB_PARAMS if param not in node]:
|
|
589
|
+
raise DatafileValidationError(
|
|
590
|
+
f"Some DynamoDB params have been provided, but the following required ones are missing: {missing}"
|
|
591
|
+
)
|
|
578
592
|
# Validate S3 params
|
|
579
|
-
|
|
593
|
+
elif any(param in node for param in BLOB_STORAGE_PARAMS):
|
|
580
594
|
if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
|
|
581
595
|
raise DatafileValidationError(
|
|
582
596
|
f"Some connection params have been provided, but the following required ones are missing: {missing}"
|
|
@@ -2097,6 +2111,8 @@ def parse(
|
|
|
2097
2111
|
"s3_arn": assign_var("s3_arn"),
|
|
2098
2112
|
"s3_access_key": assign_var("s3_access_key"),
|
|
2099
2113
|
"s3_secret": assign_var("s3_secret"),
|
|
2114
|
+
"dynamodb_arn": assign_var("dynamodb_arn"),
|
|
2115
|
+
"dynamodb_region": assign_var("dynamodb_region"),
|
|
2100
2116
|
"gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
|
|
2101
2117
|
"gcs_access_id": assign_var("gcs_hmac_access_id"),
|
|
2102
2118
|
"gcs_secret": assign_var("gcs_hmac_secret"),
|
|
@@ -219,7 +219,9 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
|
|
|
219
219
|
{"name": "processed_messages", "type": "Int32"},
|
|
220
220
|
{"name": "processed_bytes", "type": "Int32"},
|
|
221
221
|
{"name": "committed_messages", "type": "Int32"},
|
|
222
|
+
{"name": "time_read", "type": "Float32"},
|
|
222
223
|
{"name": "time_process", "type": "Float32"},
|
|
224
|
+
{"name": "time_write", "type": "Float32"},
|
|
223
225
|
{"name": "msg", "type": "String"},
|
|
224
226
|
],
|
|
225
227
|
},
|
|
@@ -880,7 +882,9 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
|
|
|
880
882
|
{"name": "processed_messages", "type": "Int32"},
|
|
881
883
|
{"name": "processed_bytes", "type": "Int32"},
|
|
882
884
|
{"name": "committed_messages", "type": "Int32"},
|
|
885
|
+
{"name": "time_read", "type": "Float32"},
|
|
883
886
|
{"name": "time_process", "type": "Float32"},
|
|
887
|
+
{"name": "time_write", "type": "Float32"},
|
|
884
888
|
{"name": "msg", "type": "String"},
|
|
885
889
|
],
|
|
886
890
|
},
|
|
@@ -398,6 +398,14 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
|
|
|
398
398
|
>>> parse_table_structure('c Nullable(Float32) DEFAULT NULL')
|
|
399
399
|
[{'name': 'c', 'type': 'Float32', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': True, 'normalized_name': 'c'}]
|
|
400
400
|
|
|
401
|
+
>>> parse_table_structure('c LowCardinality(Nullable(String))')
|
|
402
|
+
[{'name': 'c', 'type': 'LowCardinality(Nullable(String))', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
|
|
403
|
+
|
|
404
|
+
>>> parse_table_structure('c Nullable(LowCardinality(Nullable(String)))')
|
|
405
|
+
Traceback (most recent call last):
|
|
406
|
+
...
|
|
407
|
+
ValueError: Nested type LowCardinality(Nullable(String)) cannot be inside Nullable type
|
|
408
|
+
|
|
401
409
|
>>> parse_table_structure("c String DEFAULT 'bla'")
|
|
402
410
|
[{'name': 'c', 'type': 'String', 'codec': None, 'default_value': "DEFAULT 'bla'", 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
|
|
403
411
|
|
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '4.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '4.6.0'
|
|
8
|
+
__revision__ = '1d9f802'
|
|
@@ -1010,16 +1010,29 @@ class TinyB:
|
|
|
1010
1010
|
kafka_sasl_mechanism="PLAIN",
|
|
1011
1011
|
kafka_security_protocol="SASL_SSL",
|
|
1012
1012
|
kafka_ssl_ca_pem=None,
|
|
1013
|
+
kafka_sasl_oauthbearer_method=None,
|
|
1014
|
+
kafka_sasl_oauthbearer_aws_region=None,
|
|
1015
|
+
kafka_sasl_oauthbearer_aws_role_arn=None,
|
|
1016
|
+
kafka_sasl_oauthbearer_aws_external_id=None,
|
|
1013
1017
|
):
|
|
1018
|
+
is_oauthbearer = kafka_sasl_mechanism == "OAUTHBEARER"
|
|
1014
1019
|
params = {
|
|
1015
1020
|
"service": "kafka",
|
|
1016
1021
|
"kafka_security_protocol": kafka_security_protocol,
|
|
1017
1022
|
"kafka_sasl_mechanism": kafka_sasl_mechanism,
|
|
1018
1023
|
"kafka_bootstrap_servers": kafka_bootstrap_servers,
|
|
1019
|
-
"kafka_sasl_plain_username": kafka_key,
|
|
1020
|
-
"kafka_sasl_plain_password": kafka_secret,
|
|
1021
1024
|
"name": kafka_connection_name,
|
|
1022
1025
|
}
|
|
1026
|
+
if is_oauthbearer:
|
|
1027
|
+
params["kafka_sasl_oauthbearer_method"] = kafka_sasl_oauthbearer_method
|
|
1028
|
+
params["kafka_sasl_oauthbearer_aws_region"] = kafka_sasl_oauthbearer_aws_region
|
|
1029
|
+
params["kafka_sasl_oauthbearer_aws_role_arn"] = kafka_sasl_oauthbearer_aws_role_arn
|
|
1030
|
+
# external_id is optional; the server fills it in from the workspace if absent.
|
|
1031
|
+
if kafka_sasl_oauthbearer_aws_external_id:
|
|
1032
|
+
params["kafka_sasl_oauthbearer_aws_external_id"] = kafka_sasl_oauthbearer_aws_external_id
|
|
1033
|
+
else:
|
|
1034
|
+
params["kafka_sasl_plain_username"] = kafka_key
|
|
1035
|
+
params["kafka_sasl_plain_password"] = kafka_secret
|
|
1023
1036
|
|
|
1024
1037
|
if kafka_schema_registry_url:
|
|
1025
1038
|
params["kafka_schema_registry_url"] = kafka_schema_registry_url
|
|
@@ -1170,6 +1183,21 @@ class TinyB:
|
|
|
1170
1183
|
except Exception:
|
|
1171
1184
|
return False
|
|
1172
1185
|
|
|
1186
|
+
def validate_dynamodb(
|
|
1187
|
+
self, table_arn: str, region: str, role_arn: str, external_id_seed: Optional[str] = None
|
|
1188
|
+
) -> Dict[str, Any]:
|
|
1189
|
+
body: Dict[str, Any] = {"table_arn": table_arn, "region": region, "role_arn": role_arn}
|
|
1190
|
+
if external_id_seed:
|
|
1191
|
+
body["external_id_seed"] = external_id_seed
|
|
1192
|
+
response = self._req_raw(
|
|
1193
|
+
"/v1/integrations/dynamodb/validate",
|
|
1194
|
+
method="POST",
|
|
1195
|
+
data=json.dumps(body),
|
|
1196
|
+
)
|
|
1197
|
+
if response.status_code >= 400:
|
|
1198
|
+
raise requests.HTTPError(parse_error_response(response), response=response)
|
|
1199
|
+
return response.json()
|
|
1200
|
+
|
|
1173
1201
|
def get_trust_policy(self, service: str, external_id_seed: Optional[str] = None) -> Dict[str, Any]:
|
|
1174
1202
|
params = {}
|
|
1175
1203
|
if external_id_seed:
|
|
@@ -1182,10 +1210,15 @@ class TinyB:
|
|
|
1182
1210
|
params["bucket"] = bucket
|
|
1183
1211
|
return self._req(f"/v0/integrations/{service}/policies/write-access-policy?{urlencode(params)}")
|
|
1184
1212
|
|
|
1185
|
-
def get_access_read_policy(
|
|
1213
|
+
def get_access_read_policy(
|
|
1214
|
+
self, service: str, bucket: Optional[str] = None, table_name: Optional[str] = None
|
|
1215
|
+
) -> Dict[str, Any]:
|
|
1186
1216
|
params = {}
|
|
1187
1217
|
if bucket:
|
|
1188
|
-
|
|
1218
|
+
# The Kafka endpoint scopes the policy via `msk_cluster_arn`, not `bucket`.
|
|
1219
|
+
params["msk_cluster_arn" if service == "kafka" else "bucket"] = bucket
|
|
1220
|
+
if table_name:
|
|
1221
|
+
params["table_name"] = table_name
|
|
1189
1222
|
return self._req(f"/v0/integrations/{service}/policies/read-access-policy?{urlencode(params)}")
|
|
1190
1223
|
|
|
1191
1224
|
def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:
|
|
@@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Tuple
|
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
|
|
10
|
-
from tinybird.tb.modules.cli import cli
|
|
10
|
+
from tinybird.tb.modules.cli import _looks_like_uuid, cli, ensure_valid_workspace_name
|
|
11
11
|
from tinybird.tb.modules.common import (
|
|
12
12
|
MAIN_BRANCH,
|
|
13
13
|
create_workspace_branch,
|
|
@@ -93,7 +93,8 @@ def branch_ls(sort: bool) -> None:
|
|
|
93
93
|
help="Wait for data branch jobs to finish, showing a progress bar. Disabled by default.",
|
|
94
94
|
)
|
|
95
95
|
def create_branch(branch_name: Optional[str], last_partition: bool, ignore_datasources: List[str], wait: bool) -> None:
|
|
96
|
-
|
|
96
|
+
normalized_branch_name = ensure_valid_workspace_name(branch_name) if branch_name else branch_name
|
|
97
|
+
create_workspace_branch(normalized_branch_name, last_partition, False, list(ignore_datasources), wait)
|
|
97
98
|
|
|
98
99
|
|
|
99
100
|
@branch.command(name="rm", short_help="Removes an branch from the workspace. It can't be recovered.")
|
|
@@ -194,11 +195,14 @@ def clear_branch(
|
|
|
194
195
|
if branch_name_or_id:
|
|
195
196
|
if branch_name_or_id == MAIN_BRANCH:
|
|
196
197
|
raise CLIException(FeedbackManager.error_not_allowed_in_main_branch())
|
|
198
|
+
lookup_branch_name_or_id = (
|
|
199
|
+
branch_name_or_id if _looks_like_uuid(branch_name_or_id) else ensure_valid_workspace_name(branch_name_or_id)
|
|
200
|
+
)
|
|
197
201
|
workspace_to_clear = next(
|
|
198
202
|
(
|
|
199
203
|
workspace
|
|
200
204
|
for workspace in workspace_branches
|
|
201
|
-
if workspace["name"] ==
|
|
205
|
+
if workspace["name"] == lookup_branch_name_or_id or workspace["id"] == lookup_branch_name_or_id
|
|
202
206
|
),
|
|
203
207
|
None,
|
|
204
208
|
)
|
|
@@ -210,11 +214,16 @@ def clear_branch(
|
|
|
210
214
|
raise CLIBranchException(FeedbackManager.error_not_a_branch(cli=get_cli_name()))
|
|
211
215
|
|
|
212
216
|
if not workspace_to_clear:
|
|
213
|
-
raise CLIBranchException(
|
|
217
|
+
raise CLIBranchException(
|
|
218
|
+
FeedbackManager.error_branch(
|
|
219
|
+
branch=lookup_branch_name_or_id if branch_name_or_id else "",
|
|
220
|
+
cli=get_cli_name(),
|
|
221
|
+
)
|
|
222
|
+
)
|
|
214
223
|
|
|
215
224
|
branch_name = workspace_to_clear["name"]
|
|
216
225
|
if yes or click.confirm(FeedbackManager.warning_confirm_clear_workspace()):
|
|
217
|
-
was_current_branch = workspace_to_clear["id"] == config
|
|
226
|
+
was_current_branch = workspace_to_clear["id"] == config["id"]
|
|
218
227
|
client = config.get_client(token=current_main_workspace.get("token"))
|
|
219
228
|
try:
|
|
220
229
|
client.delete_branch(workspace_to_clear["id"])
|
|
@@ -247,6 +247,8 @@ def build_project(
|
|
|
247
247
|
if load_fixtures:
|
|
248
248
|
append_project_fixtures(project, tb_client, project_files)
|
|
249
249
|
echo_build_feedback(build.get("feedback", []))
|
|
250
|
+
if with_connections:
|
|
251
|
+
echo_dynamodb_local_backfill_feedback(build)
|
|
250
252
|
return True
|
|
251
253
|
elif build_result == "failed":
|
|
252
254
|
error = format_build_errors(result.get("errors", []))
|
|
@@ -461,6 +463,29 @@ def echo_build_feedback(feedback: list[dict[str, Any]]) -> None:
|
|
|
461
463
|
)
|
|
462
464
|
|
|
463
465
|
|
|
466
|
+
def echo_dynamodb_local_backfill_feedback(build: dict[str, Any]) -> None:
|
|
467
|
+
datasources_by_id = {datasource.get("id"): datasource.get("name") for datasource in build.get("datasources", [])}
|
|
468
|
+
for data_linker in build.get("data_linkers", []):
|
|
469
|
+
if data_linker.get("service") != "dynamodb":
|
|
470
|
+
continue
|
|
471
|
+
|
|
472
|
+
settings = data_linker.get("settings") or {}
|
|
473
|
+
export_arn = settings.get("initial_export_arn")
|
|
474
|
+
if not export_arn:
|
|
475
|
+
continue
|
|
476
|
+
|
|
477
|
+
datasource_name = datasources_by_id.get(data_linker.get("datasource_id")) or "unknown"
|
|
478
|
+
click.echo(
|
|
479
|
+
FeedbackManager.warning(
|
|
480
|
+
message=(
|
|
481
|
+
f"△ DynamoDB initial export backfill started for datasource '{datasource_name}'. "
|
|
482
|
+
"AWS exports can stay in progress for several minutes; Tinybird Local will keep retrying "
|
|
483
|
+
f"the import until AWS marks the export as completed. Export ARN: {export_arn}."
|
|
484
|
+
)
|
|
485
|
+
)
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
|
|
464
489
|
def format_build_errors(build_errors: list[dict[str, Any]]) -> str:
|
|
465
490
|
full_error_msg = ""
|
|
466
491
|
for build_error in build_errors:
|
|
@@ -68,6 +68,7 @@ PROJECT_TYPE_TYPESCRIPT = "ts-sdk"
|
|
|
68
68
|
PROJECT_TYPE_PYTHON = "python-sdk"
|
|
69
69
|
PROJECT_TYPE_CLI = "cli"
|
|
70
70
|
PROJECT_TYPES = {PROJECT_TYPE_TYPESCRIPT, PROJECT_TYPE_PYTHON, PROJECT_TYPE_CLI}
|
|
71
|
+
UUID_PATTERN = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE)
|
|
71
72
|
|
|
72
73
|
|
|
73
74
|
CLI_PROJECT_MARKERS = (
|
|
@@ -163,16 +164,44 @@ def is_main_git_branch(branch_name: Optional[str]) -> bool:
|
|
|
163
164
|
return branch_name in {"main", "master"}
|
|
164
165
|
|
|
165
166
|
|
|
166
|
-
def sanitize_branch_name(branch_name: str) -> str:
|
|
167
|
+
def sanitize_branch_name(branch_name: str, *, enforce_workspace_prefix_rules: bool = True) -> str:
|
|
167
168
|
sanitized = re.sub(r"[^a-zA-Z0-9_]", "_", branch_name)
|
|
168
169
|
sanitized = re.sub(r"_+", "_", sanitized)
|
|
169
|
-
|
|
170
|
+
sanitized = sanitized.strip("_")
|
|
171
|
+
if enforce_workspace_prefix_rules:
|
|
172
|
+
if sanitized and sanitized[0].isdigit():
|
|
173
|
+
sanitized = f"branch_{sanitized}"
|
|
174
|
+
if sanitized.startswith("d_"):
|
|
175
|
+
sanitized = f"branch_{sanitized}"
|
|
176
|
+
return sanitized
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _looks_like_uuid(value: str) -> bool:
|
|
180
|
+
return bool(UUID_PATTERN.fullmatch(value))
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def ensure_valid_workspace_name(name: str, *, context: str = "Branch") -> str:
|
|
184
|
+
sanitized = sanitize_branch_name(name)
|
|
185
|
+
if not sanitized:
|
|
186
|
+
raise CLIException(
|
|
187
|
+
FeedbackManager.error(
|
|
188
|
+
message=(
|
|
189
|
+
f"{context} name '{name}' is not valid. "
|
|
190
|
+
"Name must start with a letter and contain only letters, numbers, and underscores."
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
if sanitized != name:
|
|
196
|
+
click.echo(FeedbackManager.warning_branch_name_sanitized(context=context, original=name, sanitized=sanitized))
|
|
197
|
+
|
|
198
|
+
return sanitized
|
|
170
199
|
|
|
171
200
|
|
|
172
201
|
def get_tinybird_branch_name_from_git_branch(branch_name: Optional[str]) -> Optional[str]:
|
|
173
202
|
if not branch_name:
|
|
174
203
|
return None
|
|
175
|
-
sanitized = sanitize_branch_name(branch_name)
|
|
204
|
+
sanitized = sanitize_branch_name(branch_name, enforce_workspace_prefix_rules=False)
|
|
176
205
|
return sanitized or None
|
|
177
206
|
|
|
178
207
|
|
|
@@ -1105,26 +1105,57 @@ def is_url_valid(url):
|
|
|
1105
1105
|
return False
|
|
1106
1106
|
|
|
1107
1107
|
|
|
1108
|
-
def
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1108
|
+
def _parse_kafka_host_port(entry: str) -> tuple[str, int]:
|
|
1109
|
+
"""Parse a single Kafka bootstrap entry of the form 'host' or 'host:port'.
|
|
1110
|
+
|
|
1111
|
+
Raises CLIException on malformed input. Defaults the port to 9092 when omitted.
|
|
1112
|
+
"""
|
|
1113
|
+
parts = entry.split(":")
|
|
1112
1114
|
if len(parts) > 2:
|
|
1113
1115
|
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1114
1116
|
host = parts[0]
|
|
1115
1117
|
port_str = parts[1] if len(parts) == 2 else "9092"
|
|
1116
1118
|
try:
|
|
1117
|
-
|
|
1119
|
+
return host, int(port_str)
|
|
1118
1120
|
except Exception:
|
|
1119
1121
|
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def validate_kafka_bootstrap_servers(host_and_port):
|
|
1125
|
+
"""Validate a Kafka bootstrap-servers string (single entry or comma-separated list).
|
|
1126
|
+
|
|
1127
|
+
Format is checked for every entry. Connectivity is then probed per entry — Kafka
|
|
1128
|
+
only needs one reachable bootstrap broker to discover the rest of the cluster,
|
|
1129
|
+
so the validator passes as long as at least one host accepts a TCP connection.
|
|
1130
|
+
The last connection error is re-raised when every host is unreachable.
|
|
1131
|
+
"""
|
|
1132
|
+
if not isinstance(host_and_port, str):
|
|
1133
|
+
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1134
|
+
|
|
1135
|
+
entries = [e.strip() for e in host_and_port.split(",") if e.strip()]
|
|
1136
|
+
if not entries:
|
|
1137
|
+
raise CLIException(FeedbackManager.error_kafka_bootstrap_server())
|
|
1138
|
+
|
|
1139
|
+
# Format-check every entry up front so the user hears about all bad ones at
|
|
1140
|
+
# once rather than discovering them sequentially.
|
|
1141
|
+
hosts = [_parse_kafka_host_port(e) for e in entries]
|
|
1142
|
+
|
|
1143
|
+
last_error: Optional[CLIException] = None
|
|
1144
|
+
for host, port in hosts:
|
|
1145
|
+
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
|
|
1146
|
+
try:
|
|
1147
|
+
sock.settimeout(3)
|
|
1148
|
+
sock.connect((host, port))
|
|
1149
|
+
return # at least one reachable broker is enough
|
|
1150
|
+
except TimeoutError:
|
|
1151
|
+
last_error = CLIException(
|
|
1152
|
+
FeedbackManager.error_kafka_bootstrap_server_conn_timeout(host=host, port=port)
|
|
1153
|
+
)
|
|
1154
|
+
except Exception:
|
|
1155
|
+
last_error = CLIException(FeedbackManager.error_kafka_bootstrap_server_conn(host=host, port=port))
|
|
1156
|
+
|
|
1157
|
+
if last_error is not None:
|
|
1158
|
+
raise last_error
|
|
1128
1159
|
|
|
1129
1160
|
|
|
1130
1161
|
def validate_kafka_key(s):
|
|
@@ -1137,9 +1168,11 @@ def validate_kafka_secret(s):
|
|
|
1137
1168
|
raise CLIException("Password format is not correct, it must be a string")
|
|
1138
1169
|
|
|
1139
1170
|
|
|
1140
|
-
def validate_string_connector_param(param, s):
|
|
1171
|
+
def validate_string_connector_param(param: str, s: str) -> None:
|
|
1141
1172
|
if not isinstance(s, str):
|
|
1142
1173
|
raise CLIConnectionException(param + " format is not correct, it must be a string")
|
|
1174
|
+
if not s or not s.strip():
|
|
1175
|
+
raise CLIConnectionException(param + " cannot be empty")
|
|
1143
1176
|
|
|
1144
1177
|
|
|
1145
1178
|
def validate_connection_name(client, connection_name, service):
|
|
@@ -1590,7 +1623,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1590
1623
|
local_unavailable: bool = False,
|
|
1591
1624
|
) -> Tuple[str, str, Optional[TinyB], Optional[TinyB]]:
|
|
1592
1625
|
"""
|
|
1593
|
-
Run the interactive AWS IAM Role connection flow for S3.
|
|
1626
|
+
Run the interactive AWS IAM Role connection flow for S3 or DynamoDB.
|
|
1594
1627
|
|
|
1595
1628
|
Guides the user through creating an IAM policy and role with the appropriate
|
|
1596
1629
|
trust policy that includes AWS account IDs from the selected environments.
|
|
@@ -1598,7 +1631,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1598
1631
|
Args:
|
|
1599
1632
|
config: The CLI configuration dictionary.
|
|
1600
1633
|
client: The TinyB client instance.
|
|
1601
|
-
service: The data connector service type (e.g., 's3').
|
|
1634
|
+
service: The data connector service type (e.g., 's3' or 'dynamodb').
|
|
1602
1635
|
connection_name: The name for the connection being created.
|
|
1603
1636
|
policy: The access policy type ('read' or 'write').
|
|
1604
1637
|
local_unavailable: If True, local environment is unavailable (e.g., missing AWS credentials).
|
|
@@ -1606,23 +1639,41 @@ def run_aws_iamrole_connection_flow(
|
|
|
1606
1639
|
Returns:
|
|
1607
1640
|
A tuple containing:
|
|
1608
1641
|
- role_arn (str): The AWS IAM Role ARN entered by the user.
|
|
1609
|
-
- region (str): The AWS region where the
|
|
1642
|
+
- region (str): The AWS region where the resource is located.
|
|
1610
1643
|
- cloud_client (Optional[TinyB]): The TinyB client instance for the cloud environment.
|
|
1611
1644
|
- local_client (Optional[TinyB]): The TinyB client instance for the local environment.
|
|
1612
1645
|
"""
|
|
1646
|
+
bucket_name: Optional[str]
|
|
1647
|
+
table_name: Optional[str] = None
|
|
1648
|
+
region_prompt: str
|
|
1613
1649
|
if service == DataConnectorType.AMAZON_DYNAMODB:
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
)
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1650
|
+
table_name = click.prompt(
|
|
1651
|
+
FeedbackManager.highlight(
|
|
1652
|
+
message="? DynamoDB table name (specific name recommended, use '*' for unrestricted access in IAM policy)"
|
|
1653
|
+
),
|
|
1654
|
+
prompt_suffix="\n> ",
|
|
1655
|
+
)
|
|
1656
|
+
validate_string_connector_param("DynamoDB table name", table_name)
|
|
1657
|
+
bucket_name = click.prompt(
|
|
1658
|
+
FeedbackManager.highlight(
|
|
1659
|
+
message="? Export bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
|
|
1660
|
+
),
|
|
1661
|
+
prompt_suffix="\n> ",
|
|
1662
|
+
)
|
|
1663
|
+
validate_string_connector_param("Export bucket name", bucket_name)
|
|
1664
|
+
region_prompt = "? Region (the region where the DynamoDB table is located)"
|
|
1665
|
+
else:
|
|
1666
|
+
bucket_name = click.prompt(
|
|
1667
|
+
FeedbackManager.highlight(
|
|
1668
|
+
message="? Bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
|
|
1669
|
+
),
|
|
1670
|
+
prompt_suffix="\n> ",
|
|
1671
|
+
)
|
|
1672
|
+
validate_string_connector_param("Bucket", bucket_name)
|
|
1673
|
+
region_prompt = "? Region (the region where the bucket is located)"
|
|
1623
1674
|
|
|
1624
1675
|
region = click.prompt(
|
|
1625
|
-
FeedbackManager.highlight(message=
|
|
1676
|
+
FeedbackManager.highlight(message=region_prompt),
|
|
1626
1677
|
default="us-east-1",
|
|
1627
1678
|
show_default=True,
|
|
1628
1679
|
prompt_suffix="\n> ",
|
|
@@ -1705,15 +1756,27 @@ def run_aws_iamrole_connection_flow(
|
|
|
1705
1756
|
# Use cloud_client as the main client if local is unavailable
|
|
1706
1757
|
policy_client = cloud_client if local_unavailable and cloud_client else client
|
|
1707
1758
|
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1759
|
+
if table_name is not None:
|
|
1760
|
+
access_policy, trust_policy, _ = get_aws_iamrole_policies(
|
|
1761
|
+
policy_client,
|
|
1762
|
+
service=service,
|
|
1763
|
+
policy=policy,
|
|
1764
|
+
bucket=bucket_name,
|
|
1765
|
+
table_name=table_name,
|
|
1766
|
+
external_id_seed=connection_name,
|
|
1767
|
+
cloud_client=cloud_client if not local_unavailable else None,
|
|
1768
|
+
local_client=local_client,
|
|
1769
|
+
)
|
|
1770
|
+
else:
|
|
1771
|
+
access_policy, trust_policy, _ = get_aws_iamrole_policies(
|
|
1772
|
+
policy_client,
|
|
1773
|
+
service=service,
|
|
1774
|
+
policy=policy,
|
|
1775
|
+
bucket=bucket_name,
|
|
1776
|
+
external_id_seed=connection_name,
|
|
1777
|
+
cloud_client=cloud_client if not local_unavailable else None,
|
|
1778
|
+
local_client=local_client,
|
|
1779
|
+
)
|
|
1717
1780
|
|
|
1718
1781
|
click.echo(FeedbackManager.gray(message="\n» Step 1: AWS Authentication"))
|
|
1719
1782
|
click.echo(
|
|
@@ -1723,7 +1786,10 @@ def run_aws_iamrole_connection_flow(
|
|
|
1723
1786
|
)
|
|
1724
1787
|
click.echo(
|
|
1725
1788
|
FeedbackManager.info(
|
|
1726
|
-
message=
|
|
1789
|
+
message=(
|
|
1790
|
+
"You'll be creating a single IAM Policy and Role to access your data. "
|
|
1791
|
+
"Using IAM Roles improves security by providing temporary credentials and following least privilege principles."
|
|
1792
|
+
)
|
|
1727
1793
|
)
|
|
1728
1794
|
)
|
|
1729
1795
|
click.echo(FeedbackManager.click_enter_to_continue())
|
|
@@ -1748,7 +1814,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1748
1814
|
click.echo(FeedbackManager.info(message="3. Copy and paste the following policy:"))
|
|
1749
1815
|
click.echo(FeedbackManager.highlight(message=f"\n{access_policy}\n"))
|
|
1750
1816
|
click.echo(
|
|
1751
|
-
FeedbackManager.info(message=f"4. Name the policy something meaningful (e.g.,
|
|
1817
|
+
FeedbackManager.info(message=f"4. Name the policy something meaningful (e.g., TinybirdAccess-{bucket_name})")
|
|
1752
1818
|
)
|
|
1753
1819
|
click.echo(FeedbackManager.info(message="5. Click 'Create policy'"))
|
|
1754
1820
|
click.echo(FeedbackManager.click_enter_to_continue())
|
|
@@ -1774,7 +1840,7 @@ def run_aws_iamrole_connection_flow(
|
|
|
1774
1840
|
click.echo(FeedbackManager.highlight(message=f"\n{trust_policy}\n"))
|
|
1775
1841
|
click.echo(FeedbackManager.info(message="4. Click Next, search for and select the policy you just created"))
|
|
1776
1842
|
click.echo(
|
|
1777
|
-
FeedbackManager.info(message=f"5. Name the role something meaningful (e.g.,
|
|
1843
|
+
FeedbackManager.info(message=f"5. Name the role something meaningful (e.g., TinybirdRole-{bucket_name})")
|
|
1778
1844
|
)
|
|
1779
1845
|
click.echo(FeedbackManager.info(message="6. Click 'Create role'"))
|
|
1780
1846
|
click.echo(FeedbackManager.info(message="7. Copy the Role ARN from the role details page"))
|
|
@@ -1921,6 +1987,7 @@ def get_aws_iamrole_policies(
|
|
|
1921
1987
|
service: str,
|
|
1922
1988
|
policy: str = "write",
|
|
1923
1989
|
bucket: Optional[str] = None,
|
|
1990
|
+
table_name: Optional[str] = None,
|
|
1924
1991
|
external_id_seed: Optional[str] = None,
|
|
1925
1992
|
cloud_client: Optional[TinyB] = None,
|
|
1926
1993
|
local_client: Optional[TinyB] = None,
|
|
@@ -1946,7 +2013,7 @@ def get_aws_iamrole_policies(
|
|
|
1946
2013
|
if policy == "write":
|
|
1947
2014
|
access_policy = client.get_access_write_policy(service, bucket)
|
|
1948
2015
|
elif policy == "read":
|
|
1949
|
-
access_policy = client.get_access_read_policy(service, bucket)
|
|
2016
|
+
access_policy = client.get_access_read_policy(service, bucket, table_name=table_name)
|
|
1950
2017
|
else:
|
|
1951
2018
|
raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
|
|
1952
2019
|
if not len(access_policy) > 0:
|