tinybird-cli 3.3.1.dev2__tar.gz → 3.3.1.dev4__tar.gz

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.3.1.dev2
+Version: 3.3.1.dev4
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,10 +19,21 @@ Changelog
 
 ---------
 
+
+3.3.1.dev4
+***********
+
+- `Changed` script `exec_test.sh`  from `tb init --git` to run retries to avoid flakiness in tests
+
+3.3.1.dev3
+***********
+
+- `Fixed` `IMPORT` and `ENGINE` changes were not detected `tb deploy`
+
 3.3.1.dev2
 ***********
 
-- `Changed` Update `tb release rm --oldest-rollback` to use new api
+- `Changed` Update `tb release rm --oldest-rollback` to use new API
 
 3.3.1.dev1
 ************
@@ -31,7 +42,7 @@ Changelog
 
 3.3.0
 ************
-- `Added` Show warning when doing data or pipe operations directly against Live
+- `Added` Show warning when doing data or pipe operations directly against Live.
 - `Added` `bi_stats`, `bi_stats_rt` and `endpoint_errors` to the list of service data sources that the CLI ignore when looking for dependencies of pipes being pushed.
 - `Added` S3 Iam Role connection creation
 - `Added` Connector credential validation

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://docs.tinybird.co/cli.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.3.1.dev2'
-__revision__ = 'aa7ddaa'
+__version__ = '3.3.1.dev4'
+__revision__ = '4ea51cf'

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/tinybird/connector_settings.py

@@ -5,7 +5,12 @@ from typing import List, Optional, Union
 
 from pydantic import BaseModel
 
-from tinybird.data_connectors.credentials import GCSConnectorCredentials, IAMRoleAWSCredentials, S3ConnectorCredentials
+from tinybird.data_connectors.credentials import (
+    GCSConnectorCredentials,
+    GCSServiceAccountCredentials,
+    IAMRoleAWSCredentials,
+    S3ConnectorCredentials,
+)
 
 GCS_BASE_URL = "https://storage.googleapis.com"
 
@@ -17,6 +22,7 @@ class DataConnectorType(str, Enum):
     BIGQUERY = "bigquery"
     GCLOUD_STORAGE = "gcs"
     GCLOUD_STORAGE_HMAC = "gcs_hmac"
+    GCLOUD_STORAGE_SA = "gcs_service_account"
     AMAZON_S3 = "s3"
     AMAZON_S3_IAMROLE = "s3_iamrole"
 
@@ -107,6 +113,18 @@ class BigQueryConnectorSetting(DataConnectorSetting):
     account: Optional[str] = None
 
 
+class GCSServiceAccountConnectorSetting(DataConnectorSetting):
+    account_email: str
+    endpoint_url: str = GCS_BASE_URL
+
+    @property
+    def credentials(self) -> GCSServiceAccountCredentials:
+        return GCSServiceAccountCredentials(account_email=self.account_email)
+
+    def get_provider_name(self) -> str:
+        return DataConnectorType.GCLOUD_STORAGE
+
+
 class GCSHmacConnectorSetting(DataConnectorSetting):
     gcs_hmac_access_id: str
     gcs_hmac_secret: str
@@ -139,11 +157,14 @@ DATA_CONNECTOR_SETTINGS: dict[DataConnectors, type[DataConnectorSetting]] = {
     DataConnectors.BIGQUERY: BigQueryConnectorSetting,
     DataConnectors.GCLOUD_STORAGE: GCSConnectorSetting,
     DataConnectors.GCLOUD_STORAGE_HMAC: GCSHmacConnectorSetting,
+    DataConnectors.GCLOUD_STORAGE_SA: GCSServiceAccountConnectorSetting,
     DataConnectors.AMAZON_S3: S3ConnectorSetting,
     DataConnectors.AMAZON_S3_IAMROLE: S3IAMConnectorSetting,
 }
 
-SinkConnectorSettings = Union[S3ConnectorSetting, S3IAMConnectorSetting, GCSHmacConnectorSetting]
+SinkConnectorSettings = Union[
+    S3ConnectorSetting, S3IAMConnectorSetting, GCSHmacConnectorSetting, GCSServiceAccountConnectorSetting
+]
 
 
 class DataLinkerSettings:
@@ -188,6 +209,13 @@ class DataSinkSettings:
         "format",
         "compression",
     ]
+    gcs_service_account = [
+        "bucket_path",
+        "file_template",
+        "partition_node",
+        "format",
+        "compression",
+    ]
 
 
 class DataSensitiveSettings:

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/tinybird/data_connectors/credentials.py

@@ -24,3 +24,8 @@ class IAMRoleAWSCredentials(ConnectorCredentials):
 class GCSConnectorCredentials(ConnectorCredentials):
     access_key_id: str
     secret_access_key: str
+
+
+@dataclasses.dataclass(frozen=True)
+class GCSServiceAccountCredentials(ConnectorCredentials):
+    account_email: str

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/tinybird/datafile.py

@@ -4277,8 +4277,11 @@ async def format_datasource(
             format_description(file_parts, doc)
         format_tokens(file_parts, doc)
         format_schema(file_parts, doc.nodes[0])
-        await format_engine(file_parts, doc.nodes[0], only_ttl=True, client=client)
+        await format_engine(file_parts, doc.nodes[0], only_ttl=True if not for_deploy_diff else False, client=client)
+        if for_deploy_diff:
+            format_import_settings(file_parts, doc.nodes[0])
         format_shared_with(file_parts, doc)
+
     else:
         format_sources(file_parts, doc)
         format_maintainer(file_parts, doc)

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/tinybird/tb_cli_modules/cicd.py

@@ -221,23 +221,43 @@ run_test() {
     fi
     echo "** $(cat $t)"
     tmpfile=$(mktemp)
-    if bash $t $2 >$tmpfile; then
-        if diff -B ${t}.result $tmpfile; then
-            echo "Test $t: OK";
+    retries=0
+    TOTAL_RETRIES=3
+
+    # When appending fixtures, we need to retry in case of the data is not replicated in time
+    while [ $retries -lt $TOTAL_RETRIES ]; do
+        # Run the test and store the output in a temporary file
+        bash $t $2 >$tmpfile
+        exit_code=$?
+        if [ "$exit_code" -eq 0 ]; then
+            # If the test passed, break the loop
+            if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+                break
+            # If the test failed, increment the retries counter and try again
+            else
+                retries=$((retries+1))
+            fi
+        # If the bash command failed, print an error message and break the loop
         else
-            echo "🚨 ERROR: Test $t failed, diff:";
-            diff -B ${t}.result $tmpfile
-            cat $tmpfile
-            rm $tmpfile
-            return 1
+            break
         fi
+    done
+
+    if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then
+        echo "✅ Test $t passed"
+        rm $tmpfile
+        return 0
+    elif [ $retries -eq $TOTAL_RETRIES ]; then
+        echo "🚨 ERROR: Test $t failed, diff:";
+        diff -B ${t}.result $tmpfile
+        rm $tmpfile
+        return 1
     else
         echo "🚨 ERROR: Test $t failed with bash command exit code $?"
         cat $tmpfile
         rm $tmpfile
         return 1
     fi
-    rm $tmpfile
     echo ""
 }
 export -f run_test

{tinybird-cli-3.3.1.dev2 → tinybird-cli-3.3.1.dev4}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.3.1.dev2
+Version: 3.3.1.dev4
 Summary: Tinybird Command Line Tool
 Home-page: https://docs.tinybird.co/cli.html
 Author: Tinybird
@@ -19,10 +19,21 @@ Changelog
 
 ---------
 
+
+3.3.1.dev4
+***********
+
+- `Changed` script `exec_test.sh`  from `tb init --git` to run retries to avoid flakiness in tests
+
+3.3.1.dev3
+***********
+
+- `Fixed` `IMPORT` and `ENGINE` changes were not detected `tb deploy`
+
 3.3.1.dev2
 ***********
 
-- `Changed` Update `tb release rm --oldest-rollback` to use new api
+- `Changed` Update `tb release rm --oldest-rollback` to use new API
 
 3.3.1.dev1
 ************
@@ -31,7 +42,7 @@ Changelog
 
 3.3.0
 ************
-- `Added` Show warning when doing data or pipe operations directly against Live
+- `Added` Show warning when doing data or pipe operations directly against Live.
 - `Added` `bi_stats`, `bi_stats_rt` and `endpoint_errors` to the list of service data sources that the CLI ignore when looking for dependencies of pipes being pushed.
 - `Added` S3 Iam Role connection creation
 - `Added` Connector credential validation