tinybird-cli 3.7.1.dev1__tar.gz → 3.7.1.dev2__tar.gz

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.7.1.dev1
+Version: 3.7.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -19,6 +19,17 @@ Changelog
 
 ---------
 
+3.7.1.dev2
+************
+
+- `Changed` Upgrade clickhouse-toolset to 0.30dev0
+- `Added` Support for S3IAM role for ingest
+
+3.7.1.dev1
+************
+
+- `Changed` Decouple cli from connector_settings module
+
 3.7.0
 ************
 

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli/introduction.html'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '3.7.1.dev1'
-__revision__ = 'f5f887d'
+__version__ = '3.7.1.dev2'
+__revision__ = '7afe9ab'

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/tinybird/datafile.py

@@ -120,7 +120,7 @@ class CopyParameters:
     COPY_SCHEDULE = "copy_schedule"
 
 
-PREVIEW_CONNECTOR_SERVICES = ["s3", "gcs"]
+PREVIEW_CONNECTOR_SERVICES = ["s3", "s3_iamrole", "gcs"]
 
 ON_DEMAND = "@on-demand"
 
@@ -1401,7 +1401,7 @@ async def process_file(
         params.update(get_engine_params(node))
 
         if "import_service" in node or "import_connection_name" in node:
-            VALID_SERVICES: Tuple[str, ...] = ("bigquery", "snowflake", "s3", "gcs")
+            VALID_SERVICES: Tuple[str, ...] = ("bigquery", "snowflake", "s3", "s3_iamrole", "gcs")
 
             import_params = await get_import_params(params, node)
 
@@ -3106,6 +3106,7 @@ async def new_ds(
             "bigquery": ["service", "cron", "external_data_source"],
             "snowflake": ["connector", "service", "cron", "external_data_source"],
             "s3": ["connector", "service", "cron", "bucket_uri"],
+            "s3_iamrole": ["connector", "service", "cron", "bucket_uri"],
             "gcs": ["connector", "service", "cron", "bucket_uri"],
         }.get(service, [])
 

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/tinybird/tb_cli_modules/connection.py

@@ -6,21 +6,16 @@
 import json
 import os
 import sys
+from enum import Enum
 from os import getcwd
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 import click
 import pyperclip
 from click import Context
 
 from tinybird.client import DoesNotExistException, TinyB
-from tinybird.connector_settings import (
-    CLI_EXCLUDED_SETTINGS,
-    DATA_CONNECTOR_SETTINGS,
-    DataConnectors,
-    DataSensitiveSettings,
-)
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.cli import cli
 from tinybird.tb_cli_modules.common import (
@@ -40,6 +35,82 @@ from tinybird.tb_cli_modules.exceptions import CLIConnectionException
 from tinybird.tb_cli_modules.telemetry import is_ci_environment
 
 
+class DataConnectorType(str, Enum):
+    KAFKA = "kafka"
+    GCLOUD_SCHEDULER = "gcscheduler"
+    SNOWFLAKE = "snowflake"
+    BIGQUERY = "bigquery"
+    GCLOUD_STORAGE = "gcs"
+    GCLOUD_STORAGE_HMAC = "gcs_hmac"
+    GCLOUD_STORAGE_SA = "gcs_service_account"
+    AMAZON_S3 = "s3"
+    AMAZON_S3_IAMROLE = "s3_iamrole"
+
+    def __str__(self) -> str:
+        return self.value
+
+
+DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
+    DataConnectorType.KAFKA: [
+        "kafka_bootstrap_servers",
+        "kafka_sasl_plain_username",
+        "kafka_sasl_plain_password",
+        "cli_version",
+        "endpoint",
+        "kafka_security_protocol",
+        "kafka_sasl_mechanism",
+        "kafka_schema_registry_url",
+    ],
+    DataConnectorType.GCLOUD_SCHEDULER: ["gcscheduler_region"],
+    DataConnectorType.SNOWFLAKE: [
+        "account",
+        "username",
+        "password",
+        "role",
+        "warehouse",
+        "warehouse_size",
+        "stage",
+        "integration",
+    ],
+    DataConnectorType.BIGQUERY: ["account"],
+    DataConnectorType.GCLOUD_STORAGE: [
+        "gcs_private_key_id",
+        "gcs_client_x509_cert_url",
+        "gcs_project_id",
+        "gcs_client_id",
+        "gcs_client_email",
+        "gcs_private_key",
+    ],
+    DataConnectorType.GCLOUD_STORAGE_HMAC: [
+        "gcs_hmac_access_id",
+        "gcs_hmac_secret",
+    ],
+    DataConnectorType.GCLOUD_STORAGE_SA: ["account_email"],
+    DataConnectorType.AMAZON_S3: [
+        "s3_access_key_id",
+        "s3_secret_access_key",
+        "s3_region",
+    ],
+    DataConnectorType.AMAZON_S3_IAMROLE: [
+        "s3_iamrole_arn",
+        "s3_iamrole_region",
+        "s3_iamrole_external_id",
+    ],
+}
+
+SENSITIVE_CONNECTOR_SETTINGS = {
+    DataConnectorType.KAFKA: ["kafka_sasl_plain_password"],
+    DataConnectorType.GCLOUD_SCHEDULER: [
+        "gcscheduler_target_url",
+        "gcscheduler_job_name",
+        "gcscheduler_region",
+    ],
+    DataConnectorType.GCLOUD_STORAGE_HMAC: ["gcs_hmac_secret"],
+    DataConnectorType.AMAZON_S3: ["s3_secret_access_key"],
+    DataConnectorType.AMAZON_S3_IAMROLE: ["s3_iamrole_arn"],
+}
+
+
 @cli.group()
 @click.pass_context
 def connection(ctx: Context) -> None:
@@ -348,7 +419,7 @@ async def connection_rm(ctx: Context, connection_id: str, force: bool) -> None:
 @click.option("--connector", help="Filter by connector")
 @click.pass_context
 @coro
-async def connection_ls(ctx: Context, connector: Optional[DataConnectors] = None) -> None:
+async def connection_ls(ctx: Context, connector: Optional[DataConnectorType] = None) -> None:
     """List connections."""
     obj: Dict[str, Any] = ctx.ensure_object(dict)
     client: TinyB = obj["client"]
@@ -363,12 +434,10 @@ async def connection_ls(ctx: Context, connector: Optional[DataConnectors] = None
         sensitive_settings = []
         columns = ["service", "name", "id", "connected_datasources"]
     else:
-        sensitive_settings = getattr(DataSensitiveSettings, connector)
+        sensitive_settings = SENSITIVE_CONNECTOR_SETTINGS.get(connector, [])
         columns = ["service", "name", "id", "connected_datasources"]
         if connector_settings := DATA_CONNECTOR_SETTINGS.get(connector):
-            settings = connector_settings.model_fields.keys()
-            excluded_settings = CLI_EXCLUDED_SETTINGS.get(connector, [])
-            columns += [key.replace("tb_", "") for key in settings if key not in excluded_settings]
+            columns += connector_settings
 
     for connection in connections:
         row = [_get_setting_value(connection, setting, sensitive_settings) for setting in columns]
@@ -727,7 +796,7 @@ async def connection_create_s3_iamrole(
     async def create_connection(connection_name, role_arn, region) -> None:
         conn_file_name = f"{connection_name}.connection"
         conn_file_path = Path(getcwd(), conn_file_name)
-        service = DataConnectors.AMAZON_S3_IAMROLE
+        service = DataConnectorType.AMAZON_S3_IAMROLE
 
         if os.path.isfile(conn_file_path):
             raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/tinybird_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tinybird-cli
-Version: 3.7.1.dev1
+Version: 3.7.1.dev2
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli/introduction.html
 Author: Tinybird
@@ -19,6 +19,17 @@ Changelog
 
 ---------
 
+3.7.1.dev2
+************
+
+- `Changed` Upgrade clickhouse-toolset to 0.30dev0
+- `Added` Support for S3IAM role for ingest
+
+3.7.1.dev1
+************
+
+- `Changed` Decouple cli from connector_settings module
+
 3.7.0
 ************
 

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/tinybird_cli.egg-info/SOURCES.txt

@@ -2,7 +2,6 @@ tinybird/__cli__.py
 tinybird/check_pypi.py
 tinybird/client.py
 tinybird/config.py
-tinybird/connector_settings.py
 tinybird/connectors.py
 tinybird/context.py
 tinybird/datafile.py
@@ -18,7 +17,6 @@ tinybird/tb_cli.py
 tinybird/tornado_template.py
 tinybird/ch_utils/constants.py
 tinybird/ch_utils/engine.py
-tinybird/data_connectors/credentials.py
 tinybird/tb_cli_modules/auth.py
 tinybird/tb_cli_modules/branch.py
 tinybird/tb_cli_modules/cicd.py

{tinybird-cli-3.7.1.dev1 → tinybird-cli-3.7.1.dev2}/tinybird_cli.egg-info/requires.txt

@@ -1,5 +1,5 @@
 click<8.2,>=8.1.6
-clickhouse-toolset==0.29.dev0
+clickhouse-toolset==0.30.dev0
 colorama==0.4.6
 cryptography>=3.4.8
 croniter==1.3.8

tinybird-cli-3.7.1.dev1/tinybird/connector_settings.py

@@ -1,239 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-from typing import List, Optional, Union
-
-from pydantic import BaseModel
-
-from tinybird.data_connectors.credentials import (
-    GCSConnectorCredentials,
-    GCSServiceAccountCredentials,
-    IAMRoleAWSCredentials,
-    S3ConnectorCredentials,
-)
-
-GCS_BASE_URL = "https://storage.googleapis.com"
-
-
-class DataConnectorType(str, Enum):
-    KAFKA = "kafka"
-    GCLOUD_SCHEDULER = "gcscheduler"
-    SNOWFLAKE = "snowflake"
-    BIGQUERY = "bigquery"
-    GCLOUD_STORAGE = "gcs"
-    GCLOUD_STORAGE_HMAC = "gcs_hmac"
-    GCLOUD_STORAGE_SA = "gcs_service_account"
-    AMAZON_S3 = "s3"
-    AMAZON_S3_IAMROLE = "s3_iamrole"
-
-    def __str__(self) -> str:
-        return self.value
-
-
-DataConnectors = DataConnectorType  # this is just to make picking happy
-
-
-class DataConnectorSetting(BaseModel):
-    def get_region(self) -> str:
-        return "unknown"
-
-    def get_provider_name(self) -> str:
-        return "unknown"
-
-
-class KafkaConnectorSetting(DataConnectorSetting):
-    kafka_bootstrap_servers: str
-    kafka_sasl_plain_username: str
-    kafka_sasl_plain_password: str
-    cli_version: Optional[str] = None
-    tb_endpoint: Optional[str] = None
-    kafka_security_protocol: Optional[str] = None
-    kafka_sasl_mechanism: Optional[str] = None
-    kafka_schema_registry_url: Optional[str] = None
-
-
-class S3ConnectorSetting(DataConnectorSetting):
-    s3_access_key_id: str
-    s3_secret_access_key: str
-    s3_region: str
-    endpoint_url: Optional[str] = None
-
-    @property
-    def credentials(self) -> S3ConnectorCredentials:
-        return S3ConnectorCredentials(
-            access_key_id=self.s3_access_key_id,
-            secret_access_key=self.s3_secret_access_key,
-            region=self.s3_region,
-        )
-
-    def get_provider_name(self) -> str:
-        return DataConnectorType.AMAZON_S3
-
-    def get_region(self) -> str:
-        return self.s3_region
-
-
-class S3IAMConnectorSetting(DataConnectorSetting):
-    s3_iamrole_arn: str
-    s3_iamrole_region: str
-    s3_iamrole_external_id: str
-    endpoint_url: Optional[str] = None
-
-    @property
-    def credentials(self) -> IAMRoleAWSCredentials:
-        return IAMRoleAWSCredentials(
-            role_arn=self.s3_iamrole_arn,
-            external_id=self.s3_iamrole_external_id,
-            region=self.s3_iamrole_region,
-        )
-
-    def get_provider_name(self) -> str:
-        return DataConnectorType.AMAZON_S3
-
-    def get_region(self) -> str:
-        return self.s3_iamrole_region
-
-
-class SnowflakeConnectorSetting(DataConnectorSetting):
-    account: str
-    username: str
-    password: str
-    role: str
-    warehouse: str
-    warehouse_size: Optional[str] = None
-    stage: Optional[str] = None
-    integration: Optional[str] = None
-
-
-class GCSchedulerConnectorSetting(DataConnectorSetting):
-    gcscheduler_region: Optional[str] = None
-
-
-class BigQueryConnectorSetting(DataConnectorSetting):
-    account: Optional[str] = None
-
-
-class GCSServiceAccountConnectorSetting(DataConnectorSetting):
-    account_email: str
-    endpoint_url: str = GCS_BASE_URL
-
-    @property
-    def credentials(self) -> GCSServiceAccountCredentials:
-        return GCSServiceAccountCredentials(account_email=self.account_email)
-
-    def get_provider_name(self) -> str:
-        return DataConnectorType.GCLOUD_STORAGE
-
-
-class GCSHmacConnectorSetting(DataConnectorSetting):
-    gcs_hmac_access_id: str
-    gcs_hmac_secret: str
-    endpoint_url: str = GCS_BASE_URL
-
-    @property
-    def credentials(self) -> GCSConnectorCredentials:
-        return GCSConnectorCredentials(
-            access_key_id=self.gcs_hmac_access_id,
-            secret_access_key=self.gcs_hmac_secret,
-        )
-
-    def get_provider_name(self) -> str:
-        return DataConnectorType.GCLOUD_STORAGE
-
-
-class GCSConnectorSetting(DataConnectorSetting):
-    gcs_private_key_id: str
-    gcs_client_x509_cert_url: str
-    gcs_project_id: str
-    gcs_client_id: str
-    gcs_client_email: str
-    gcs_private_key: str
-
-
-DATA_CONNECTOR_SETTINGS: dict[DataConnectors, type[DataConnectorSetting]] = {
-    DataConnectors.KAFKA: KafkaConnectorSetting,
-    DataConnectors.GCLOUD_SCHEDULER: GCSchedulerConnectorSetting,
-    DataConnectors.SNOWFLAKE: SnowflakeConnectorSetting,
-    DataConnectors.BIGQUERY: BigQueryConnectorSetting,
-    DataConnectors.GCLOUD_STORAGE: GCSConnectorSetting,
-    DataConnectors.GCLOUD_STORAGE_HMAC: GCSHmacConnectorSetting,
-    DataConnectors.GCLOUD_STORAGE_SA: GCSServiceAccountConnectorSetting,
-    DataConnectors.AMAZON_S3: S3ConnectorSetting,
-    DataConnectors.AMAZON_S3_IAMROLE: S3IAMConnectorSetting,
-}
-
-SinkConnectorSettings = Union[
-    S3ConnectorSetting, S3IAMConnectorSetting, GCSHmacConnectorSetting, GCSServiceAccountConnectorSetting
-]
-
-
-class DataLinkerSettings:
-    kafka = [
-        "tb_datasource",
-        "tb_token",
-        "kafka_topic",
-        "kafka_group_id",
-        "kafka_auto_offset_reset",
-        "kafka_store_raw_value",
-        "kafka_store_headers",
-    ]
-
-
-class DataSinkSettings:
-    gcscheduler = [
-        "cron",
-        "timezone",
-        "status",
-        "gcscheduler_target_url",
-        "gcscheduler_job_name",
-        "gcscheduler_region",
-    ]
-    gcs_hmac = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-    s3 = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-    s3_iamrole = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-    gcs_service_account = [
-        "bucket_path",
-        "file_template",
-        "partition_node",
-        "format",
-        "compression",
-    ]
-
-
-class DataSensitiveSettings:
-    kafka = ["kafka_sasl_plain_password"]
-    gcscheduler = [
-        "gcscheduler_target_url",
-        "gcscheduler_job_name",
-        "gcscheduler_region",
-    ]
-    bigquery: List[str] = []
-    snowflake: List[str] = []
-    gcs_hmac: List[str] = ["gcs_hmac_secret"]
-    s3: List[str] = ["s3_secret_access_key"]
-    s3_iamrole: List[str] = ["s3_iamrole_arn"]
-
-
-CLI_EXCLUDED_SETTINGS = {
-    "gcs_hmac": ["endpoint_url"],
-    "s3": ["endpoint_url"],
-    "s3_iamrole": ["endpoint_url"],
-}

tinybird-cli-3.7.1.dev1/tinybird/data_connectors/credentials.py

@@ -1,31 +0,0 @@
-import abc
-import dataclasses
-
-
-class ConnectorCredentials(abc.ABC):
-    pass
-
-
-@dataclasses.dataclass(frozen=True)
-class S3ConnectorCredentials(ConnectorCredentials):
-    access_key_id: str
-    secret_access_key: str
-    region: str
-
-
-@dataclasses.dataclass(frozen=True)
-class IAMRoleAWSCredentials(ConnectorCredentials):
-    role_arn: str
-    external_id: str
-    region: str
-
-
-@dataclasses.dataclass(frozen=True)
-class GCSConnectorCredentials(ConnectorCredentials):
-    access_key_id: str
-    secret_access_key: str
-
-
-@dataclasses.dataclass(frozen=True)
-class GCSServiceAccountCredentials(ConnectorCredentials):
-    account_email: str