tinybird 4.5.12.dev0__tar.gz → 4.6.1__tar.gz

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.5.12.dev0
+Version: 4.6.1
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -27,7 +27,7 @@ Requires-Dist: requests<3,>=2.28.1
 Requires-Dist: shandy-sqlfmt==0.11.1
 Requires-Dist: shandy-sqlfmt[jinjafmt]==0.11.1
 Requires-Dist: toposort==1.10
-Requires-Dist: tornado~=6.0.0
+Requires-Dist: tornado~=6.5.5
 Requires-Dist: urllib3<2,>=1.26.14
 Requires-Dist: watchdog==6.0.0
 Requires-Dist: wheel
@@ -52,12 +52,21 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.6.1
+*******
+
+- `Changed` `tb pull` to overwrite existing local files when pulling from a Tinybird workspace.
+
+4.6.0
+*******
+
+- `Added` Support for AWS DynamoDB datasource creation and management.
+
 4.5.12
 *******
 
 - `Fixed` `tb info` now lists branches and marks the current branch correctly when running on a branch.
 
-
 4.5.11
 *******
 

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/datafile/common.py

@@ -277,6 +277,13 @@ VALID_BLOB_STORAGE_CRON_VALUES = {
     "@auto",
 }
 
+REQUIRED_DYNAMODB_PARAMS = {
+    "import_connection_name",
+    "import_table_arn",
+    "import_export_bucket",
+}
+DYNAMODB_PARAMS = REQUIRED_DYNAMODB_PARAMS
+
 
 def extract_column_names_from_sorting_key_part(part: str) -> List[str]:
     """
@@ -575,8 +582,15 @@ class Datafile:
                 )
             if "kafka_group_id" in node and not str(node["kafka_group_id"]).strip():
                 raise DatafileValidationError("KAFKA_GROUP_ID cannot be empty")
+            # Validate DynamoDB params first since import_connection_name is shared with blob storage.
+            is_dynamodb = any(param in node for param in ("import_table_arn", "import_export_bucket"))
+            if is_dynamodb:
+                if missing := [param for param in REQUIRED_DYNAMODB_PARAMS if param not in node]:
+                    raise DatafileValidationError(
+                        f"Some DynamoDB params have been provided, but the following required ones are missing: {missing}"
+                    )
             # Validate S3 params
-            if any(param in node for param in BLOB_STORAGE_PARAMS):
+            elif any(param in node for param in BLOB_STORAGE_PARAMS):
                 if missing := [param for param in REQUIRED_BLOB_STORAGE_PARAMS if param not in node]:
                     raise DatafileValidationError(
                         f"Some connection params have been provided, but the following required ones are missing: {missing}"
@@ -2097,6 +2111,8 @@ def parse(
             "s3_arn": assign_var("s3_arn"),
             "s3_access_key": assign_var("s3_access_key"),
             "s3_secret": assign_var("s3_secret"),
+            "dynamodb_arn": assign_var("dynamodb_arn"),
+            "dynamodb_region": assign_var("dynamodb_region"),
             "gcs_service_account_credentials_json": assign_var_json("gcs_service_account_credentials_json"),
             "gcs_access_id": assign_var("gcs_hmac_access_id"),
             "gcs_secret": assign_var("gcs_hmac_secret"),

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/service_datasources.py

@@ -219,7 +219,9 @@ def get_tinybird_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "processed_messages", "type": "Int32"},
                 {"name": "processed_bytes", "type": "Int32"},
                 {"name": "committed_messages", "type": "Int32"},
+                {"name": "time_read", "type": "Float32"},
                 {"name": "time_process", "type": "Float32"},
+                {"name": "time_write", "type": "Float32"},
                 {"name": "msg", "type": "String"},
             ],
         },
@@ -880,7 +882,9 @@ def get_organization_service_datasources() -> List[Dict[str, Any]]:
                 {"name": "processed_messages", "type": "Int32"},
                 {"name": "processed_bytes", "type": "Int32"},
                 {"name": "committed_messages", "type": "Int32"},
+                {"name": "time_read", "type": "Float32"},
                 {"name": "time_process", "type": "Float32"},
+                {"name": "time_write", "type": "Float32"},
                 {"name": "msg", "type": "String"},
             ],
         },

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/sql.py

@@ -398,6 +398,14 @@ def parse_table_structure(schema: str) -> List[Dict[str, Any]]:
     >>> parse_table_structure('c Nullable(Float32) DEFAULT NULL')
     [{'name': 'c', 'type': 'Float32', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': True, 'normalized_name': 'c'}]
 
+    >>> parse_table_structure('c LowCardinality(Nullable(String))')
+    [{'name': 'c', 'type': 'LowCardinality(Nullable(String))', 'codec': None, 'default_value': None, 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
+
+    >>> parse_table_structure('c Nullable(LowCardinality(Nullable(String)))')
+    Traceback (most recent call last):
+    ...
+    ValueError: Nested type LowCardinality(Nullable(String)) cannot be inside Nullable type
+
     >>> parse_table_structure("c String DEFAULT 'bla'")
     [{'name': 'c', 'type': 'String', 'codec': None, 'default_value': "DEFAULT 'bla'", 'jsonpath': None, 'nullable': False, 'normalized_name': 'c'}]
 

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/sql_template.py

@@ -7,7 +7,7 @@ from collections import deque
 from datetime import datetime
 from functools import lru_cache
 from json import loads
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 from tornado import escape
 from tornado.util import ObjectDict, exec_in, unicode_type
@@ -1499,7 +1499,7 @@ def generate(self, **kwargs) -> Tuple[str, TemplateExecutionResults]:
     )
 
     exec_in(self.compiled, namespace)
-    execute = namespace["_tt_execute"]
+    execute = cast(Callable[[], bytes], namespace["_tt_execute"])
     # Clear the traceback module's cache of source data now that
     # we've generated a new template (mainly for this module's
     # unittests, where different tests reuse the same name).

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/tb/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/forward/commands'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '4.5.12.dev0'
-__revision__ = '6a70694'
+__version__ = '4.6.1'
+__revision__ = '41da03f'

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/tb/client.py

@@ -1183,6 +1183,21 @@ class TinyB:
         except Exception:
             return False
 
+    def validate_dynamodb(
+        self, table_arn: str, region: str, role_arn: str, external_id_seed: Optional[str] = None
+    ) -> Dict[str, Any]:
+        body: Dict[str, Any] = {"table_arn": table_arn, "region": region, "role_arn": role_arn}
+        if external_id_seed:
+            body["external_id_seed"] = external_id_seed
+        response = self._req_raw(
+            "/v1/integrations/dynamodb/validate",
+            method="POST",
+            data=json.dumps(body),
+        )
+        if response.status_code >= 400:
+            raise requests.HTTPError(parse_error_response(response), response=response)
+        return response.json()
+
     def get_trust_policy(self, service: str, external_id_seed: Optional[str] = None) -> Dict[str, Any]:
         params = {}
         if external_id_seed:
@@ -1195,11 +1210,15 @@ class TinyB:
             params["bucket"] = bucket
         return self._req(f"/v0/integrations/{service}/policies/write-access-policy?{urlencode(params)}")
 
-    def get_access_read_policy(self, service: str, bucket: Optional[str] = None) -> Dict[str, Any]:
+    def get_access_read_policy(
+        self, service: str, bucket: Optional[str] = None, table_name: Optional[str] = None
+    ) -> Dict[str, Any]:
         params = {}
         if bucket:
             # The Kafka endpoint scopes the policy via `msk_cluster_arn`, not `bucket`.
             params["msk_cluster_arn" if service == "kafka" else "bucket"] = bucket
+        if table_name:
+            params["table_name"] = table_name
         return self._req(f"/v0/integrations/{service}/policies/read-access-policy?{urlencode(params)}")
 
     def sql_get_format(self, sql: str, with_clickhouse_format: bool = False) -> str:

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/tb/modules/build_common.py

@@ -247,6 +247,8 @@ def build_project(
             if load_fixtures:
                 append_project_fixtures(project, tb_client, project_files)
             echo_build_feedback(build.get("feedback", []))
+            if with_connections:
+                echo_dynamodb_local_backfill_feedback(build)
             return True
         elif build_result == "failed":
             error = format_build_errors(result.get("errors", []))
@@ -461,6 +463,29 @@ def echo_build_feedback(feedback: list[dict[str, Any]]) -> None:
         )
 
 
+def echo_dynamodb_local_backfill_feedback(build: dict[str, Any]) -> None:
+    datasources_by_id = {datasource.get("id"): datasource.get("name") for datasource in build.get("datasources", [])}
+    for data_linker in build.get("data_linkers", []):
+        if data_linker.get("service") != "dynamodb":
+            continue
+
+        settings = data_linker.get("settings") or {}
+        export_arn = settings.get("initial_export_arn")
+        if not export_arn:
+            continue
+
+        datasource_name = datasources_by_id.get(data_linker.get("datasource_id")) or "unknown"
+        click.echo(
+            FeedbackManager.warning(
+                message=(
+                    f"△ DynamoDB initial export backfill started for datasource '{datasource_name}'. "
+                    "AWS exports can stay in progress for several minutes; Tinybird Local will keep retrying "
+                    f"the import until AWS marks the export as completed. Export ARN: {export_arn}."
+                )
+            )
+        )
+
+
 def format_build_errors(build_errors: list[dict[str, Any]]) -> str:
     full_error_msg = ""
     for build_error in build_errors:

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/tb/modules/common.py

@@ -105,7 +105,9 @@ def echo_safe_humanfriendly_tables_format_smart_table(data: Iterable[Any], colum
     try:
         click.echo(humanfriendly.tables.format_smart_table(data, column_names=column_names))
     except ValueError as exc:
-        if str(exc) == "max() arg is an empty sequence":
+        # Python 3.11 wording: "max() arg is an empty sequence"
+        # Python 3.12 wording: "max() iterable argument is empty"
+        if str(exc) in ("max() arg is an empty sequence", "max() iterable argument is empty"):
             click.echo("------------")
             click.echo("Empty")
             click.echo("------------")
@@ -122,7 +124,9 @@ def echo_safe_humanfriendly_tables_format_pretty_table(data: Iterable[Any], colu
     try:
         click.echo(humanfriendly.tables.format_pretty_table(data, column_names=column_names))
     except ValueError as exc:
-        if str(exc) == "max() arg is an empty sequence":
+        # Python 3.11 wording: "max() arg is an empty sequence"
+        # Python 3.12 wording: "max() iterable argument is empty"
+        if str(exc) in ("max() arg is an empty sequence", "max() iterable argument is empty"):
             click.echo("------------")
             click.echo("Empty")
             click.echo("------------")
@@ -139,7 +143,9 @@ def echo_safe_format_table(data: Iterable[Any], columns) -> None:
     try:
         click.echo(format_table(data, columns))
     except ValueError as exc:
-        if str(exc) == "max() arg is an empty sequence":
+        # Python 3.11 wording: "max() arg is an empty sequence"
+        # Python 3.12 wording: "max() iterable argument is empty"
+        if str(exc) in ("max() arg is an empty sequence", "max() iterable argument is empty"):
             click.echo("------------")
             click.echo("Empty")
             click.echo("------------")
@@ -1623,7 +1629,7 @@ def run_aws_iamrole_connection_flow(
     local_unavailable: bool = False,
 ) -> Tuple[str, str, Optional[TinyB], Optional[TinyB]]:
     """
-    Run the interactive AWS IAM Role connection flow for S3.
+    Run the interactive AWS IAM Role connection flow for S3 or DynamoDB.
 
     Guides the user through creating an IAM policy and role with the appropriate
     trust policy that includes AWS account IDs from the selected environments.
@@ -1631,7 +1637,7 @@ def run_aws_iamrole_connection_flow(
     Args:
         config: The CLI configuration dictionary.
         client: The TinyB client instance.
-        service: The data connector service type (e.g., 's3').
+        service: The data connector service type (e.g., 's3' or 'dynamodb').
         connection_name: The name for the connection being created.
         policy: The access policy type ('read' or 'write').
         local_unavailable: If True, local environment is unavailable (e.g., missing AWS credentials).
@@ -1639,23 +1645,41 @@ def run_aws_iamrole_connection_flow(
     Returns:
         A tuple containing:
             - role_arn (str): The AWS IAM Role ARN entered by the user.
-            - region (str): The AWS region where the bucket is located.
+            - region (str): The AWS region where the resource is located.
             - cloud_client (Optional[TinyB]): The TinyB client instance for the cloud environment.
             - local_client (Optional[TinyB]): The TinyB client instance for the local environment.
     """
+    bucket_name: Optional[str]
+    table_name: Optional[str] = None
+    region_prompt: str
     if service == DataConnectorType.AMAZON_DYNAMODB:
-        raise NotImplementedError("DynamoDB is not supported")
-
-    bucket_name = click.prompt(
-        FeedbackManager.highlight(
-            message="? Bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
-        ),
-        prompt_suffix="\n> ",
-    )
-    validate_string_connector_param("Bucket", bucket_name)
+        table_name = click.prompt(
+            FeedbackManager.highlight(
+                message="? DynamoDB table name (specific name recommended, use '*' for unrestricted access in IAM policy)"
+            ),
+            prompt_suffix="\n> ",
+        )
+        validate_string_connector_param("DynamoDB table name", table_name)
+        bucket_name = click.prompt(
+            FeedbackManager.highlight(
+                message="? Export bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
+            ),
+            prompt_suffix="\n> ",
+        )
+        validate_string_connector_param("Export bucket name", bucket_name)
+        region_prompt = "? Region (the region where the DynamoDB table is located)"
+    else:
+        bucket_name = click.prompt(
+            FeedbackManager.highlight(
+                message="? Bucket name (specific name recommended, use '*' for unrestricted access in IAM policy)"
+            ),
+            prompt_suffix="\n> ",
+        )
+        validate_string_connector_param("Bucket", bucket_name)
+        region_prompt = "? Region (the region where the bucket is located)"
 
     region = click.prompt(
-        FeedbackManager.highlight(message="? Region (the region where the bucket is located)"),
+        FeedbackManager.highlight(message=region_prompt),
         default="us-east-1",
         show_default=True,
         prompt_suffix="\n> ",
@@ -1738,15 +1762,27 @@ def run_aws_iamrole_connection_flow(
     # Use cloud_client as the main client if local is unavailable
     policy_client = cloud_client if local_unavailable and cloud_client else client
 
-    access_policy, trust_policy, _ = get_aws_iamrole_policies(
-        policy_client,
-        service=service,
-        policy=policy,
-        bucket=bucket_name,
-        external_id_seed=connection_name,
-        cloud_client=cloud_client if not local_unavailable else None,
-        local_client=local_client,
-    )
+    if table_name is not None:
+        access_policy, trust_policy, _ = get_aws_iamrole_policies(
+            policy_client,
+            service=service,
+            policy=policy,
+            bucket=bucket_name,
+            table_name=table_name,
+            external_id_seed=connection_name,
+            cloud_client=cloud_client if not local_unavailable else None,
+            local_client=local_client,
+        )
+    else:
+        access_policy, trust_policy, _ = get_aws_iamrole_policies(
+            policy_client,
+            service=service,
+            policy=policy,
+            bucket=bucket_name,
+            external_id_seed=connection_name,
+            cloud_client=cloud_client if not local_unavailable else None,
+            local_client=local_client,
+        )
 
     click.echo(FeedbackManager.gray(message="\n» Step 1: AWS Authentication"))
     click.echo(
@@ -1756,7 +1792,10 @@ def run_aws_iamrole_connection_flow(
     )
     click.echo(
         FeedbackManager.info(
-            message="You'll be creating a single IAM Policy and Role to access your S3 data. Using IAM Roles improves security by providing temporary credentials and following least privilege principles."
+            message=(
+                "You'll be creating a single IAM Policy and Role to access your data. "
+                "Using IAM Roles improves security by providing temporary credentials and following least privilege principles."
+            )
         )
     )
     click.echo(FeedbackManager.click_enter_to_continue())
@@ -1781,7 +1820,7 @@ def run_aws_iamrole_connection_flow(
         click.echo(FeedbackManager.info(message="3. Copy and paste the following policy:"))
     click.echo(FeedbackManager.highlight(message=f"\n{access_policy}\n"))
     click.echo(
-        FeedbackManager.info(message=f"4. Name the policy something meaningful (e.g., TinybirdS3Access-{bucket_name})")
+        FeedbackManager.info(message=f"4. Name the policy something meaningful (e.g., TinybirdAccess-{bucket_name})")
     )
     click.echo(FeedbackManager.info(message="5. Click 'Create policy'"))
     click.echo(FeedbackManager.click_enter_to_continue())
@@ -1807,7 +1846,7 @@ def run_aws_iamrole_connection_flow(
     click.echo(FeedbackManager.highlight(message=f"\n{trust_policy}\n"))
     click.echo(FeedbackManager.info(message="4. Click Next, search for and select the policy you just created"))
     click.echo(
-        FeedbackManager.info(message=f"5. Name the role something meaningful (e.g., TinybirdS3Role-{bucket_name})")
+        FeedbackManager.info(message=f"5. Name the role something meaningful (e.g., TinybirdRole-{bucket_name})")
     )
     click.echo(FeedbackManager.info(message="6. Click 'Create role'"))
     click.echo(FeedbackManager.info(message="7. Copy the Role ARN from the role details page"))
@@ -1954,6 +1993,7 @@ def get_aws_iamrole_policies(
     service: str,
     policy: str = "write",
     bucket: Optional[str] = None,
+    table_name: Optional[str] = None,
     external_id_seed: Optional[str] = None,
     cloud_client: Optional[TinyB] = None,
     local_client: Optional[TinyB] = None,
@@ -1979,7 +2019,7 @@ def get_aws_iamrole_policies(
         if policy == "write":
             access_policy = client.get_access_write_policy(service, bucket)
         elif policy == "read":
-            access_policy = client.get_access_read_policy(service, bucket)
+            access_policy = client.get_access_read_policy(service, bucket, table_name=table_name)
         else:
             raise Exception(f"Access policy {policy} not supported. Choose from 'read' or 'write'")
         if not len(access_policy) > 0:

{tinybird-4.5.12.dev0 → tinybird-4.6.1}/tinybird/tb/modules/connection.py

@@ -20,6 +20,7 @@ from tinybird.tb.modules.common import (
     get_gcs_svc_account_creds,
     run_gcp_svc_account_connection_flow,
 )
+from tinybird.tb.modules.connection_dynamodb import connection_create_dynamodb
 from tinybird.tb.modules.connection_kafka import (
     connection_create_kafka,
     echo_kafka_data,
@@ -248,6 +249,51 @@ def connection_create_gcs(ctx: Context) -> None:
     )
 
 
+@connection_create.command(
+    name="dynamodb", short_help="Creates a AWS DynamoDB connection using IAM role authentication."
+)
+@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
+@click.option("--table-arn", default=None, help="Optional. Validate the connection against this DynamoDB table ARN")
+@click.pass_context
+def connection_create_dynamodb_cmd(ctx: Context, connection_name: Optional[str], table_arn: Optional[str]) -> None:
+    """
+    Creates a AWS DynamoDB connection using IAM role authentication.
+
+    \b
+    $ tb connection create dynamodb
+    """
+    obj: Dict[str, Any] = ctx.ensure_object(dict)
+    project: Project = obj["project"]
+    client: TinyB = obj["client"]
+    env: str = obj["env"]
+    config = obj["config"]
+
+    local_aws_unavailable = env == "local" and not client.check_aws_credentials()
+
+    if env == "local" and not local_aws_unavailable:
+        click.echo(FeedbackManager.gray(message="» Building project before continue..."))
+        error = build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
+        if error:
+            click.echo(FeedbackManager.error(message=error))
+        else:
+            click.echo(FeedbackManager.success(message="✓ Build completed"))
+
+    result = connection_create_dynamodb(ctx, connection_name=connection_name, table_arn=table_arn)
+
+    if env == "local" and not local_aws_unavailable and not result["error"]:
+        click.echo(FeedbackManager.gray(message="» Building project to access the new connection..."))
+        error = build_project(project=project, tb_client=client, watch=False, config=config, silent=True)
+        if error:
+            click.echo(FeedbackManager.error(message=error))
+        else:
+            click.echo(FeedbackManager.success(message="✓ Build completed"))
+
+    # connection_create_dynamodb already prints the failure details; exit non-zero so scripted/CI
+    # usage does not treat a connection with missing secrets as a success.
+    if result["error"]:
+        ctx.exit(1)
+
+
 @connection_create.command(name="kafka", help="Create a Kafka connection.")
 @click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
 @click.option("--bootstrap-servers", default=None, help="Kafka Bootstrap Server in form mykafka.mycloud.com:9092")