tinybird-cli 5.22.3.dev0__py3-none-any.whl → 6.0.2.dev0__py3-none-any.whl

tinybird/sql_toolset.py

@@ -149,12 +149,11 @@ def format_where_for_mutation_command(where_clause: str) -> str:
 # Functions that take table/dictionary names as string literal arguments.
 # Normalizing these would cause incorrect cache hits since different table names
 # would map to the same cache key.
-# See: https://clickhouse.com/docs/en/sql-reference/functions/other-functions#joinget
-#      https://clickhouse.com/docs/en/sql-reference/functions/ext-dict-functions
+# See: https://clickhouse.com/docs/en/sql-reference/functions/ext-dict-functions
 #      https://clickhouse.com/docs/en/sql-reference/table-functions/cluster
 #      https://clickhouse.com/docs/en/sql-reference/table-functions/remote
 _FUNCTIONS_WITH_TABLE_NAME_ARGS = re.compile(
-    r"\b(?:joinGet|joinGetOrNull|dictGet\w*|dictHas|dictIsIn|hasColumnInTable|remote|cluster|clusterAllReplicas)\s*\(",
+    r"\b(?:dictGet\w*|dictHas|dictIsIn|hasColumnInTable|remote|cluster|clusterAllReplicas)\s*\(",
     re.IGNORECASE,
 )
 
@@ -166,7 +165,7 @@ def _normalize_sql_for_cache(sql: str) -> str:
     while preserving table/column names, so queries with the same structure share
     cache entries.
 
-    However, some functions like joinGet(), dictGet*, remote(), cluster(), and
+    However, some functions like dictGet*, remote(), cluster(), and
     clusterAllReplicas() take table/dictionary names as arguments. Normalizing these
     would incorrectly map different tables to the same cache key, so we fall back to
     using the original SQL for such queries.
@@ -177,8 +176,6 @@ def _normalize_sql_for_cache(sql: str) -> str:
     'SELECT * FROM events WHERE id = ? AND name = ?'
     >>> _normalize_sql_for_cache("SELECT * FROM events")
     'SELECT * FROM events'
-    >>> _normalize_sql_for_cache("SELECT joinGet('my_table', 'col', id) FROM t")
-    "SELECT joinGet('my_table', 'col', id) FROM t"
     >>> _normalize_sql_for_cache("SELECT dictGet('my_dict', 'value', id) FROM t")
     "SELECT dictGet('my_dict', 'value', id) FROM t"
     >>> _normalize_sql_for_cache("SELECT * FROM remote('host', db, table)")
@@ -200,7 +197,7 @@ def _normalize_sql_for_cache(sql: str) -> str:
 
 # Cache for sql_get_used_tables using normalized SQL as key.
 # Uses lru-dict (C extension) for a fast LRU implementation.
-_sql_get_used_tables_cache: LRU = LRU(2**15)
+_sql_get_used_tables_cache: LRU = LRU(2**13)
 _sql_get_used_tables_cache_lock = threading.Lock()
 _sql_get_used_tables_cache_hits = 0
 _sql_get_used_tables_cache_misses = 0
@@ -395,38 +392,15 @@ class ReplacementsDict(dict):
         if isinstance(v, tuple):
             k, r = v
             if callable(r):
-                r = update_callable_signature(r)(self.enabled_table_functions)
+                r = r(self.enabled_table_functions)
                 super().__setitem__(key, (k, r))
             return k, r
         if callable(v):
-            v = update_callable_signature(v)(self.enabled_table_functions)
+            v = v(self.enabled_table_functions)
             super().__setitem__(key, v)
         return v
 
 
-def update_callable_signature(func):
-    """
-    Utility function to provide backward compatibility for callable functions
-    that don't accept the enabled_table_functions parameter.
-    """
-    if callable(func):
-
-        def wrapper(enabled_table_functions=None):
-            # Check if the function accepts the enabled_table_functions parameter
-            import inspect
-
-            sig = inspect.signature(func)
-            if len(sig.parameters) == 0:
-                # Old-style callable with no parameters
-                return func()
-            else:
-                # New-style callable that accepts enabled_table_functions
-                return func(enabled_table_functions)
-
-        return wrapper
-    return func
-
-
 def tables_or_sql(replacement: dict, table_functions=False, function_allow_list=None) -> set:
     try:
         return set(

tinybird/tb_cli_modules/auth.py

@@ -13,7 +13,6 @@ from tinybird.config import get_display_host
 from tinybird.feedback_manager import FeedbackManager
 from tinybird.tb_cli_modules.cli import cli
 from tinybird.tb_cli_modules.common import (
-    configure_connector,
     coro,
     echo_safe_humanfriendly_tables_format_smart_table,
     get_host_from_region,
@@ -35,11 +34,6 @@ from tinybird.tb_cli_modules.regions import Region
 @click.option(
     "--region", envvar="TB_REGION", help="Set region. Run 'tb auth ls' to show available regions. Overrides host."
 )
-@click.option(
-    "--connector",
-    type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
-    help="Set credentials for one of the supported connectors",
-)
 @click.option(
     "-i",
     "--interactive",
@@ -49,7 +43,7 @@ from tinybird.tb_cli_modules.regions import Region
 )
 @click.pass_context
 @coro
-async def auth(ctx: click.Context, token: str, host: str, region: str, connector: str, interactive: bool) -> None:
+async def auth(ctx: click.Context, token: str, host: str, region: str, interactive: bool) -> None:
     """Configure auth."""
 
     config: CLIConfig = CLIConfig.get_project_config()
@@ -58,10 +52,6 @@ async def auth(ctx: click.Context, token: str, host: str, region: str, connector
     if host:
         config.set_host(host)
 
-    if connector:
-        await configure_connector(connector)
-        return
-
     # Only run when doing a bare 'tb auth'
     if ctx.invoked_subcommand:
         return

tinybird/tb_cli_modules/cli.py

@@ -27,7 +27,7 @@ from tinybird.client import (
     OperationCanNotBePerformed,
     TinyB,
 )
-from tinybird.config import CURRENT_VERSION, SUPPORTED_CONNECTORS, VERSION, FeatureFlags, get_config
+from tinybird.config import CURRENT_VERSION, VERSION, FeatureFlags, get_config
 from tinybird.datafile_common import (
     AlreadyExistsException,
     CLIGitRelease,
@@ -62,7 +62,6 @@ from tinybird.tb_cli_modules.common import (
     getenv_bool,
     is_major_semver,
     is_post_semver,
-    load_connector_config,
     remove_release,
     try_update_config_with_remote,
 )
@@ -85,32 +84,6 @@ DEFAULT_PATTERNS: List[Tuple[str, Union[str, Callable[[str], str]]]] = [
 @click.option("--token", help="Use auth token, defaults to TB_TOKEN envvar, then to the .tinyb file")
 @click.option("--host", help="Use custom host, defaults to TB_HOST envvar, then to https://api.tinybird.co")
 @click.option("--semver", help="Semver of a Release to run the command. Example: 1.0.0", hidden=True)
-@click.option("--gcp-project-id", help="The Google Cloud project ID", hidden=True)
-@click.option(
-    "--gcs-bucket", help="The Google Cloud Storage bucket to write temp files when using the connectors", hidden=True
-)
-@click.option(
-    "--google-application-credentials",
-    envvar="GOOGLE_APPLICATION_CREDENTIALS",
-    help="Set GOOGLE_APPLICATION_CREDENTIALS",
-    hidden=True,
-)
-@click.option("--sf-account", help="The Snowflake Account (e.g. your-domain.west-europe.azure)", hidden=True)
-@click.option("--sf-warehouse", help="The Snowflake warehouse name", hidden=True)
-@click.option("--sf-database", help="The Snowflake database name", hidden=True)
-@click.option("--sf-schema", help="The Snowflake schema name", hidden=True)
-@click.option("--sf-role", help="The Snowflake role name", hidden=True)
-@click.option("--sf-user", help="The Snowflake user name", hidden=True)
-@click.option("--sf-password", help="The Snowflake password", hidden=True)
-@click.option(
-    "--sf-storage-integration",
-    help="The Snowflake GCS storage integration name (leave empty to auto-generate one)",
-    hidden=True,
-)
-@click.option("--sf-stage", help="The Snowflake GCS stage name (leave empty to auto-generate one)", hidden=True)
-@click.option(
-    "--with-headers", help="Flag to enable connector to export with headers", is_flag=True, default=False, hidden=True
-)
 @click.option(
     "--version-warning/--no-version-warning",
     envvar="TB_VERSION_WARNING",
@@ -127,19 +100,6 @@ async def cli(
     token: str,
     host: str,
     semver: str,
-    gcp_project_id: str,
-    gcs_bucket: str,
-    google_application_credentials: str,
-    sf_account: str,
-    sf_warehouse: str,
-    sf_database: str,
-    sf_schema: str,
-    sf_role: str,
-    sf_user: str,
-    sf_password: str,
-    sf_storage_integration: str,
-    sf_stage,
-    with_headers: bool,
     version_warning: bool,
     show_tokens: bool,
 ) -> None:
@@ -212,52 +172,10 @@ async def cli(
     if ctx.invoked_subcommand == "auth":
         return
 
-    from tinybird.connectors import create_connector
-
-    if gcp_project_id and gcs_bucket and google_application_credentials and not sf_account:
-        bq_config = {
-            "project_id": gcp_project_id,
-            "bucket_name": gcs_bucket,
-            "service_account": google_application_credentials,
-            "with_headers": with_headers,
-        }
-        ctx.ensure_object(dict)["bigquery"] = create_connector("bigquery", bq_config)
-    if (
-        sf_account
-        and sf_warehouse
-        and sf_database
-        and sf_schema
-        and sf_role
-        and sf_user
-        and sf_password
-        and gcs_bucket
-        and google_application_credentials
-        and gcp_project_id
-    ):
-        sf_config = {
-            "account": sf_account,
-            "warehouse": sf_warehouse,
-            "database": sf_database,
-            "schema": sf_schema,
-            "role": sf_role,
-            "user": sf_user,
-            "password": sf_password,
-            "storage_integration": sf_storage_integration,
-            "stage": sf_stage,
-            "bucket_name": gcs_bucket,
-            "service_account": google_application_credentials,
-            "project_id": gcp_project_id,
-            "with_headers": with_headers,
-        }
-        ctx.ensure_object(dict)["snowflake"] = create_connector("snowflake", sf_config)
-
     logging.debug("debug enabled")
 
     ctx.ensure_object(dict)["client"] = _get_tb_client(config.get("token", None), config["host"], semver)  # type: ignore[arg-type]
 
-    for connector in SUPPORTED_CONNECTORS:
-        load_connector_config(ctx, connector, debug, check_uninstalled=True)
-
 
 @cli.command()
 @click.option(

tinybird/tb_cli_modules/common.py

@@ -16,9 +16,9 @@ from contextlib import closing
 from copy import deepcopy
 from enum import Enum
 from functools import wraps
-from os import chmod, environ, getcwd, getenv
+from os import chmod, getcwd, getenv
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Literal, Optional, Set, Tuple, TypedDict, Union
+from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Set, Tuple, TypedDict, Union
 from urllib.parse import urlparse
 
 import aiofiles
@@ -35,7 +35,6 @@ from packaging.version import Version
 from tinybird.client import (
     AuthException,
     AuthNoTokenException,
-    ConnectorNothingToLoad,
     DoesNotExistException,
     JobException,
     OperationCanNotBePerformed,
@@ -46,17 +45,11 @@ from tinybird.config import (
     DEFAULT_UI_HOST,
     DEPRECATED_PROJECT_PATHS,
     PROJECT_PATHS,
-    SUPPORTED_CONNECTORS,
     VERSION,
     FeatureFlags,
     get_config,
     get_display_host,
-    write_config,
 )
-
-if TYPE_CHECKING:
-    from tinybird.connectors import Connector
-
 from tinybird.feedback_manager import FeedbackManager, warning_message
 from tinybird.git_settings import DEFAULT_TINYENV_FILE
 from tinybird.syncasync import async_to_sync
@@ -78,6 +71,9 @@ from tinybird.tb_cli_modules.telemetry import (
     is_ci_environment,
 )
 
+# Pre-compiled regex pattern for name normalization
+_PATTERN_NORMALIZE_NAME = re.compile(r"[^0-9a-zA-Z_]")
+
 SUPPORTED_FORMATS = ["csv", "ndjson", "json", "parquet"]
 OLDEST_ROLLBACK = "oldest_rollback"
 MAIN_BRANCH = "main"
@@ -89,16 +85,6 @@ def obfuscate_token(value: Optional[str]) -> Optional[str]:
     return f"{value[:4]}...{value[-8:]}"
 
 
-def create_connector(connector: str, options: Dict[str, Any]):
-    # Imported here to improve startup time when the connectors aren't used
-    from tinybird.connectors import UNINSTALLED_CONNECTORS
-    from tinybird.connectors import create_connector as _create_connector
-
-    if connector in UNINSTALLED_CONNECTORS:
-        raise CLIException(FeedbackManager.error_connector_not_installed(connector=connector))
-    return _create_connector(connector, options)
-
-
 def coro(f):
     @wraps(f)
     def wrapper(*args, **kwargs):
@@ -135,7 +121,7 @@ def echo_safe_humanfriendly_tables_format_smart_table(data: Iterable[Any], colum
 
 
 def normalize_datasource_name(s: str) -> str:
-    s = re.sub(r"[^0-9a-zA-Z_]", "_", s)
+    s = _PATTERN_NORMALIZE_NAME.sub("_", s)
     if s[0] in "0123456789":
         return "c_" + s
     return s
@@ -269,24 +255,6 @@ variable to '1' or 'true'."""
         sys.exit(exit_code)
 
 
-def load_connector_config(ctx: Context, connector_name: str, debug: bool, check_uninstalled: bool = False):
-    config_file = Path(getcwd()) / f".tinyb_{connector_name}"
-    try:
-        if connector_name not in ctx.ensure_object(dict):
-            with open(config_file) as file:
-                config = json.loads(file.read())
-            from tinybird.connectors import UNINSTALLED_CONNECTORS
-
-            if check_uninstalled and connector_name in UNINSTALLED_CONNECTORS:
-                click.echo(FeedbackManager.warning_connector_not_installed(connector=connector_name))
-                return
-            ctx.ensure_object(dict)[connector_name] = create_connector(connector_name, config)
-    except OSError:
-        if debug:
-            click.echo(f"** {connector_name} connector not configured")
-        pass
-
-
 def getenv_bool(key: str, default: bool) -> bool:
     v: Optional[str] = getenv(key)
     if v is None:
@@ -306,31 +274,26 @@ def create_tb_client(ctx: Context) -> TinyB:
     return _get_tb_client(token, host, semver=semver)
 
 
-async def _analyze(filename: str, client: TinyB, format: str, connector: Optional["Connector"] = None):
+async def _analyze(filename: str, client: TinyB, format: str):
     data: Optional[bytes] = None
-    if not connector:
-        parsed = urlparse(filename)
-        if parsed.scheme in ("http", "https"):
-            meta = await client.datasource_analyze(filename)
-        else:
-            async with aiofiles.open(filename, "rb") as file:
-                # We need to read the whole file in binary for Parquet, while for the
-                # others we just read 1KiB
-                if format == "parquet":
-                    data = await file.read()
-                else:
-                    data = await file.read(1024 * 1024)
-
-            meta = await client.datasource_analyze_file(data)
+    parsed = urlparse(filename)
+    if parsed.scheme in ("http", "https"):
+        meta = await client.datasource_analyze(filename)
     else:
-        meta = connector.datasource_analyze(filename)
+        async with aiofiles.open(filename, "rb") as file:
+            # We need to read the whole file in binary for Parquet, while for the
+            # others we just read 1KiB
+            if format == "parquet":
+                data = await file.read()
+            else:
+                data = await file.read(1024 * 1024)
+
+        meta = await client.datasource_analyze_file(data)
     return meta, data
 
 
-async def _generate_datafile(
-    filename: str, client: TinyB, format: str, connector: Optional["Connector"] = None, force: Optional[bool] = False
-):
-    meta, data = await _analyze(filename, client, format, connector=connector)
+async def _generate_datafile(filename: str, client: TinyB, format: str, force: Optional[bool] = False):
+    meta, data = await _analyze(filename, client, format)
     schema = meta["analysis"]["schema"]
     schema = schema.replace(", ", ",\n    ")
     datafile = f"""DESCRIPTION >\n    Generated from {filename}\n\nSCHEMA >\n    {schema}"""
@@ -465,77 +428,12 @@ async def folder_init(
                 )
 
 
-async def configure_connector(connector):
-    if connector not in SUPPORTED_CONNECTORS:
-        raise CLIException(FeedbackManager.error_invalid_connector(connectors=", ".join(SUPPORTED_CONNECTORS)))
-
-    file_name = f".tinyb_{connector}"
-    config_file = Path(getcwd()) / file_name
-    if connector == "bigquery":
-        project = click.prompt("BigQuery project ID")
-        service_account = click.prompt(
-            "Path to a JSON service account file with permissions to export from BigQuery, write in Storage and sign URLs (leave empty to use GOOGLE_APPLICATION_CREDENTIALS environment variable)",
-            default=environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""),
-        )
-        bucket_name = click.prompt("Name of a Google Cloud Storage bucket to store temporary exported files")
-
-        try:
-            config = {"project_id": project, "service_account": service_account, "bucket_name": bucket_name}
-            await write_config(config, file_name)
-        except Exception:
-            raise CLIException(FeedbackManager.error_file_config(config_file=config_file))
-    elif connector == "snowflake":
-        sf_account = click.prompt("Snowflake Account (e.g. your-domain.west-europe.azure)")
-        sf_warehouse = click.prompt("Snowflake warehouse name")
-        sf_database = click.prompt("Snowflake database name")
-        sf_schema = click.prompt("Snowflake schema name")
-        sf_role = click.prompt("Snowflake role name")
-        sf_user = click.prompt("Snowflake user name")
-        sf_password = click.prompt("Snowflake password")
-        sf_storage_integration = click.prompt(
-            "Snowflake GCS storage integration name (leave empty to auto-generate one)", default=""
-        )
-        sf_stage = click.prompt("Snowflake GCS stage name (leave empty to auto-generate one)", default="")
-        project = click.prompt("Google Cloud project ID to store temporary files")
-        service_account = click.prompt(
-            "Path to a JSON service account file with permissions to write in Storagem, sign URLs and IAM (leave empty to use GOOGLE_APPLICATION_CREDENTIALS environment variable)",
-            default=environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""),
-        )
-        bucket_name = click.prompt("Name of a Google Cloud Storage bucket to store temporary exported files")
-
-        if not service_account:
-            service_account = getenv("GOOGLE_APPLICATION_CREDENTIALS")
-
-        try:
-            config = {
-                "account": sf_account,
-                "warehouse": sf_warehouse,
-                "database": sf_database,
-                "schema": sf_schema,
-                "role": sf_role,
-                "user": sf_user,
-                "password": sf_password,
-                "storage_integration": sf_storage_integration,
-                "stage": sf_stage,
-                "service_account": service_account,
-                "bucket_name": bucket_name,
-                "project_id": project,
-            }
-            await write_config(config, file_name)
-        except Exception:
-            raise CLIException(FeedbackManager.error_file_config(config_file=config_file))
-
-        click.echo(FeedbackManager.success_connector_config(connector=connector, file_name=file_name))
-
-
 def _compare_region_host(region_name_or_host: str, region: Dict[str, Any]) -> bool:
     if region["name"].lower() == region_name_or_host:
         return True
     if region["host"] == region_name_or_host:
         return True
-    if region["api_host"] == region_name_or_host:
-        return True
-    return False
+    return region["api_host"] == region_name_or_host
 
 
 def ask_for_region_interactively(regions):
@@ -1071,33 +969,17 @@ async def push_data(
     ctx: Context,
     datasource_name: str,
     url,
-    connector: Optional[str],
-    sql: Optional[str],
     mode: str = "append",
     sql_condition: Optional[str] = None,
     replace_options=None,
-    ignore_empty: bool = False,
     concurrency: int = 1,
 ):
     if url and type(url) is tuple:
         url = url[0]
     client: TinyB = ctx.obj["client"]
 
-    if connector and sql:
-        load_connector_config(ctx, connector, False, check_uninstalled=False)
-        if connector not in ctx.obj:
-            raise CLIException(FeedbackManager.error_connector_not_configured(connector=connector))
-        else:
-            _connector: Connector = ctx.obj[connector]
-            click.echo(FeedbackManager.info_starting_export_process(connector=connector))
-            try:
-                url = _connector.export_to_gcs(sql, datasource_name, mode)
-            except ConnectorNothingToLoad as e:
-                if ignore_empty:
-                    click.echo(str(e))
-                    return
-                else:
-                    raise e
+    if not url:
+        raise CLIException(FeedbackManager.error_missing_url(datasource=datasource_name))
 
     def cb(res):
         if cb.First:  # type: ignore[attr-defined]
@@ -1199,19 +1081,13 @@ async def push_data(
         else:
             click.echo(FeedbackManager.success_appended_datasource(datasource=datasource_name))
         click.echo(FeedbackManager.info_data_pushed(datasource=datasource_name))
-    finally:
-        try:
-            for url in urls:
-                _connector.clean(urlparse(url).path.split("/")[-1])
-        except Exception:
-            pass
 
 
 async def sync_data(ctx, datasource_name: str, yes: bool):
     client: TinyB = ctx.obj["client"]
     datasource = await client.get_datasource(datasource_name)
 
-    VALID_DATASOURCES = ["bigquery", "snowflake", "s3", "s3_iamrole", "gcs"]
+    VALID_DATASOURCES = ["s3", "s3_iamrole", "gcs"]
     if datasource["type"] not in VALID_DATASOURCES:
         raise CLIException(FeedbackManager.error_sync_not_supported(valid_datasources=VALID_DATASOURCES))
 
@@ -1296,7 +1172,7 @@ def validate_kafka_bootstrap_servers(bootstrap_servers):
             try:
                 sock.settimeout(3)
                 sock.connect((host, port))
-            except socket.timeout:
+            except TimeoutError:
                 raise CLIException(FeedbackManager.error_kafka_bootstrap_server_conn_timeout())
             except Exception:
                 raise CLIException(FeedbackManager.error_kafka_bootstrap_server_conn())
@@ -2022,7 +1898,7 @@ async def validate_aws_iamrole_connection_name(
     client: TinyB, connection_name: Optional[str], no_validate: Optional[bool] = False
 ) -> str:
     if connection_name and no_validate is False:
-        if await client.get_connector(connection_name, skip_bigquery=True) is not None:
+        if await client.get_connector(connection_name) is not None:
             raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
     else:
         while not connection_name:
@@ -2039,8 +1915,6 @@ async def validate_aws_iamrole_connection_name(
 class DataConnectorType(str, Enum):
     KAFKA = "kafka"
     GCLOUD_SCHEDULER = "gcscheduler"
-    SNOWFLAKE = "snowflake"
-    BIGQUERY = "bigquery"
     GCLOUD_STORAGE = "gcs"
     GCLOUD_STORAGE_HMAC = "gcs_hmac"
     GCLOUD_STORAGE_SA = "gcs_service_account"

tinybird/tb_cli_modules/config.py

@@ -31,9 +31,7 @@ class FeatureFlags:
     def send_telemetry(cls) -> bool:
         if os.environ.get("TB_CLI_TELEMETRY_OPTOUT", "0") == "1":
             return False
-        if "x.y.z" in CURRENT_VERSION and os.environ.get("TB_CLI_TELEMETRY_SEND_IN_LOCAL", "0") == "0":
-            return False
-        return True
+        return not ("x.y.z" in CURRENT_VERSION and os.environ.get("TB_CLI_TELEMETRY_SEND_IN_LOCAL", "0") == "0")
 
 
 def compare_versions(a: str, b: str) -> int: