tinybird-cli 6.5.6.dev0__tar.gz → 6.5.8.dev0__tar.gz

{tinybird_cli-6.5.6.dev0 → tinybird_cli-6.5.8.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird_cli
-Version: 6.5.6.dev0
+Version: 6.5.8.dev0
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/cli
 Author: Tinybird
@@ -43,6 +43,20 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+
+6.5.7
+***********
+
+- `Added` ``Markdown`` is now a valid ``FORMAT`` for SQL queries (for example ``tb sql "SELECT 1 FORMAT Markdown"``), returned as a Markdown table.
+- `Changed` ``tb datasource append`` and ``tb datasource replace`` submit remote CSV, NDJSON, and Parquet URLs through the v1 import endpoint by default.
+- `Added` Copy pipes accept ``ON_DEMAND_COMPUTE true`` so scheduled runs use on-demand compute. Manual ``tb pipe copy run`` still requires ``--on-demand-compute``, and asks for confirmation when the local datafile does not enable it (skip it with ``--yes``).
+- `Removed` The ``--main`` flag from ``tb branch regression-tests``. It ran the comparison against the main Branch for a custom CI workflow.
+
+6.5.6
+***********
+
+- `Changed` ``tb datasource append`` and ``tb datasource replace`` submit local CSV, NDJSON, and Parquet files through the v1 import endpoint by default. ``tb datasource append --events`` also submits its request body through v1. Commands wait for completion by default to preserve the existing output; pass ``--no-wait`` to print the queued job ID instead. Remote URL imports continue to use their existing route.
+
 6.5.5
 ***********
 

tinybird_cli-6.5.8.dev0/pyproject.toml

@@ -0,0 +1,246 @@
+[build-system]
+requires = [
+    "setuptools>=61.0",
+    "wheel==0.37.1",
+    "cffi~=2.0",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tinybird"
+version = "1.0"
+description = "tinybird"
+readme = { text = "tinybird analytics", content-type = "text/markdown" }
+requires-python = ">=3.13,<3.15"
+authors = [
+    { name = "tinybird.co" },
+]
+dependencies = [
+    "GitPython==3.1.44",
+    "Markdown==3.5.1",
+    "aioboto3~=15.0",
+    "aiofiles==24.1.0",
+    "aiohttp~=3.11",
+    "anthropic~=1.9",
+    "async-lru==2.3.0",
+    "aws-msk-iam-sasl-signer-python==1.0.2",
+    "cachetools~=6.0",
+    "cffi~=2.0",
+    "chardet==4.0.0",
+    "ciso8601==2.2.0",
+    "click~=8.5",
+    "clickhouse-cityhash==1.0.2.5",
+    "clickhouse-driver==0.2.9",
+    "confluent-kafka[avro,json,schemaregistry]==2.12.2",
+    "contextvars==2.4",
+    "cramjam==2.11.0",
+    "croniter==6.2.2",
+    "cryptography~=50.0",
+    "datasketch==1.6.5",
+    "dnspython==2.8.0",
+    "docker==7.1.0",
+    "fastavro==1.12.2",
+    "fastmcp==2.12.2",
+    "google-cloud-aiplatform~=1.148",
+    "google-cloud-compute==1.48.0",
+    "google-cloud-scheduler==2.20.0",
+    "google-cloud-secret-manager==2.29.0",
+    "google-cloud-storage~=3.4",
+    "google_api_core~=2.25",
+    "googleapis_common_protos==1.66.0",
+    "grpcio_status==1.67.1",
+    "humanfriendly==10.0",
+    "isal==1.8.0",
+    "jinja2",
+    "jsondiff==2.2.1",
+    "jsonschema==4.23.0",
+    "litellm~=1.97",
+    "logfire-api==4.2.0",
+    "lru-dict==1.3.0",
+    "lz4==4.4.5",
+    "memray==1.20.0",
+    "msgspec==0.21.1",
+    "multidict==6.9.1",
+    "networkx==3.4.2",
+    "numpy==1.26.4",
+    "objgraph==3.6.2",
+    "opentelemetry-api~=1.38",
+    "opentelemetry-exporter-otlp-proto-http~=1.38",
+    "opentelemetry-sdk~=1.38",
+    "orb-billing==4.80.1",
+    "orjson==3.11.9",
+    "packaging~=26.3",
+    "passlib==1.7.1",
+    "posthog==7.28.0",
+    "pre-commit==4.6.2",
+    "prometheus_client==0.26.0",
+    "prompt_toolkit==3.0.48",
+    "propcache==0.3.2",
+    "proto_plus==1.25.0",
+    "protobuf==5.29.5",
+    "psutil==7.2.2",
+    "psycopg2-binary==2.9.12",
+    "py-spy==0.4.2",
+    "pyarrow==23.0.1",
+    "pycurl==7.45.7",
+    "pydantic-ai-slim[anthropic]~=2.0",
+    "pydantic-ai-slim[google]~=2.0",
+    "pydantic-ai-slim[retries]~=2.0",
+    "pydantic~=2.12",
+    "pyjwt[crypto]==2.9.0",
+    "pympler==1.1",
+    "pynacl==1.5.0",
+    "pyperclip==1.9.0",
+    "python-dateutil==2.9.0.post0",
+    "python-dotenv==1.1.0",
+    "python-ulid==2.7.0",
+    "pyyaml~=6.0",
+    "redis[hiredis]==5.3.1",
+    "requests~=2.32.5",
+    "sentry-sdk==2.69.2",
+    "setproctitle==1.3.3",
+    "shandy-sqlfmt==0.11.1",
+    "shandy-sqlfmt[jinjafmt]==0.11.1",
+    "sqlparse==0.4.3",
+    "starlette==1.0.1",
+    "streaming-form-data==1.20.0",
+    "stripe==3.1.0",
+    "tabulate==0.8.10",
+    "thefuzz==0.22.1",
+    "tinybird-python-sdk[ai]==0.3.7",
+    "tinybird-toolset==2.7.1",
+    "toposort==1.10",
+    "tornado==6.5.5",
+    "urllib3~=2.8",
+    "uvicorn[standard]~=0.53.0",
+    "virtualenv==20.21.1",
+    "watchdog==6.0.0",
+    "wheel==0.37.1",
+    "zlib-ng==1.0.0",
+    "zstandard==0.25.0",
+]
+
+[dependency-groups]
+deploy = [
+    "GitPython==3.1.44",
+    "ansible==8.5.0",
+    "ara==1.6.1",
+    "boto3~=1.40",
+    "fabric==3.2.2",
+    "google-auth~=2.47",
+    "jsonnet",
+    "prettytable==3.8.0",
+    "requests~=2.32.5",
+    "requests[security]",
+    "xmltodict",
+]
+devtools = [
+    "flameprof",
+    "matplotlib",
+    "memory_profiler",
+    "rope",
+]
+doc = [
+    "Pallets-Sphinx-Themes==2.3.0",
+    "pyOpenSSL~=26.4",
+    "sphinx-autobuild==2024.2.4",
+    "sphinx-sitemap",
+    "sphinx==7.3.7",
+    "sphinxcontrib-httpdomain",
+    "sphinxcontrib-jquery==4.1",
+]
+lint = [
+    "botocore-stubs~=1.40",
+    "celery-types~=0.20.0",
+    "flake8~=7.4",
+    "flake8-async~=26.8",
+    "mypy==2.3.1",
+    "ruff==0.16.10",
+    "types-Markdown",
+    "types-PyYAML",
+    "types-aiofiles==24.1.0.20241221",
+    "types-boto3-lite~=1.40",
+    "types-cachetools",
+    "types-chardet",
+    "types-paramiko",
+    "types-pycurl",
+    "types-python-dateutil",
+    "types-pytz",
+    "types-requests",
+    "types-tabulate",
+    "types_croniter",
+]
+test = [
+    "coverage~=7.16",
+    "freezegun==1.5.5",
+    "google-auth~=2.47",
+    "gsutil==4.58",
+    "httpx2~=2.13",
+    "hypothesis==6.156.9",
+    "oauth2client==3.0.0",
+    "py==1.11.0",
+    "pytest-asyncio~=1.4",
+    "pytest-cov~=7.0",
+    "pytest-env==1.1.3",
+    "pytest-flakefinder==1.1.0",
+    "pytest-timeout==2.3.1",
+    "pytest-xdist==3.6.1",
+    "pytest~=8.1",
+    "rangehttpserver==1.4.0",
+    "yappi==1.7.6",
+]
+# Synced automatically by plain `uv run`/`uv sync`, no `--group` needed.
+dev = [
+    { include-group = "test" },
+    { include-group = "deploy" },
+    { include-group = "lint" },
+]
+
+[project.scripts]
+tinybird_server = "tinybird.app:run"
+tinybird_tool = "tinybird.tinybird_tool.main:cli"
+tb_old = "tinybird.tb_cli:cli"
+tb = "tinybird.tb.cli:cli"
+colibri = "tinybird.colibri.cli:cli"
+
+[tool.uv.workspace]
+members = [
+    "chbackups",
+    "clickhouse",
+    "data_connector",
+    "gatherer",
+    "platform-api",
+    "tinybird_orchestrator",
+]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+namespaces = false
+include = ["tinybird", "tinybird.*", "tinybird_shared", "tinybird_shared.*", "scripts"]
+exclude = ["tests*"]
+
+[tool.setuptools.package-data]
+tinybird = [
+    "templates/**/*",
+    "static/**/*",
+    "default_pipes/**/*",
+    "sql/**/*",
+    "monitors/**/*.sql",
+    "scripts/**/*",
+    "data_projects/**/*",
+    "data_projects/**/.*",
+    "raw_events/**/*",
+    "query_booster/**/*",
+    "deployments/**/*",
+]
+tinybird_shared = [
+    "redis_client/**/*",
+    "retry/**/*",
+    "metrics/**/*",
+    "gatherer_settings/**/*",
+    "clickhouse/**/*",
+    "gatherer_backpressure/**/*",
+    "data_connector_circuit_breaker/**/*",
+]

{tinybird_cli-6.5.6.dev0 → tinybird_cli-6.5.8.dev0}/tinybird/__cli__.py

@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
 __url__ = 'https://www.tinybird.co/docs/cli'
 __author__ = 'Tinybird'
 __author_email__ = 'support@tinybird.co'
-__version__ = '6.5.6.dev0'
-__revision__ = '5a6cd1d'
+__version__ = '6.5.8.dev0'
+__revision__ = '57adbc63'

{tinybird_cli-6.5.6.dev0 → tinybird_cli-6.5.8.dev0}/tinybird/ch_utils/constants.py

@@ -263,4 +263,5 @@ VALID_QUERY_FORMATS = (
     "JSONCompact",
     "JSONStringsEachRowWithProgress",
     "ODBCDriver2",
+    "Markdown",
 )

{tinybird_cli-6.5.6.dev0 → tinybird_cli-6.5.8.dev0}/tinybird/ch_utils/engine.py

@@ -1,3 +1,4 @@
+import functools
 import logging
 import re
 from collections import defaultdict
@@ -25,6 +26,9 @@ DEFAULT_EMPTY_PARAMETERS = ["ttl", "partition_key", "sorting_key"]
 # * `N` is a number
 SIMPLE_TTL_DEFINITION = re.compile(r"""^([a-zA-Z0-9_\-\.\(\)\ \/\*]*) \+ (toInterval[a-zA-Z]*\([0-9]+\))$""")
 
+# ClickHouse 25.8 quotes the engine name `Null`, as `null` is a keyword.
+_BACKQUOTED_ENGINE_NAME = re.compile(r"^`(\w+)`")
+
 
 class TableDetails:
     """
@@ -78,6 +82,16 @@ class TableDetails:
 
     @property
     def engine_full(self) -> Optional[str]:
+        """
+        >>> TableDetails({"engine_full": "`Null`", "engine": "Null"}).engine_full
+        'Null'
+        >>> TableDetails({"engine_full": "`Null`()", "engine": "Null"}).engine_full
+        'Null()'
+        >>> TableDetails({"engine_full": "Null", "engine": "Null"}).to_json()
+        {'engine_full': 'Null', 'engine': 'Null'}
+        >>> TableDetails({"engine_full": "`Null`", "engine": "Null"}).to_json()
+        {'engine_full': 'Null', 'engine': 'Null'}
+        """
         _engine_full: Optional[str] = self.details.get("engine_full", None)
         if not _engine_full:
             return None
@@ -86,7 +100,7 @@ class TableDetails:
         # We cannot remove index_granularity = 8192 blindly because it might be followed by other settings
         if settings and settings.strip().lower() == "index_granularity = 8192":
             _engine_full = _engine_full.replace(" SETTINGS index_granularity = 8192", "")
-        return engine_replicated_to_local(_engine_full)
+        return engine_replicated_to_local(_BACKQUOTED_ENGINE_NAME.sub(r"\1", _engine_full))
 
     @property
     def original_engine_full(self) -> Optional[str]:
@@ -111,8 +125,7 @@ class TableDetails:
 
     @property
     def version(self):
-        _version = self.details.get("version", None)
-        return _version
+        return self.details.get("version", None)
 
     def is_replicated(self) -> bool:
         engine: Optional[str] = self.details.get("engine", None)
@@ -183,6 +196,7 @@ class TableDetails:
         settings = self.details.get("settings", None)
         if settings and settings.strip().lower() != "index_granularity = 8192":
             return settings
+        return None
 
     @property
     def ttl(self):
@@ -190,23 +204,19 @@ class TableDetails:
 
     @property
     def ver(self):
-        _ver = self.details.get("ver", None)
-        return _ver
+        return self.details.get("ver", None)
 
     @property
     def is_deleted(self):
-        _is_deleted = self.details.get("is_deleted", None)
-        return _is_deleted
+        return self.details.get("is_deleted", None)
 
     @property
     def columns(self):
-        _columns = self.details.get("columns", None)
-        return _columns
+        return self.details.get("columns", None)
 
     @property
     def sign(self):
-        _sign = self.details.get("sign", None)
-        return _sign
+        return self.details.get("sign", None)
 
     @property
     def statistics(self) -> Dict[str, Any]:
@@ -444,38 +454,70 @@ def get_engine_config(engine: str):
     )
 
 
-def engine_params(columns, params: List[EngineParam], args: Dict):
-    params_values = []
+def validate_engine_config(
+    engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
+) -> None:
+    if schema is not None and columns is not None:
+        raise ValueError("You can not use 'schema' and 'columns' at the same time")
+    _, (params, options) = get_engine_config(engine)
+    schema_columns = parse_table_structure(schema) if schema is not None else (columns or [])
+    engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
+
+    for arg in engine_settings:
+        if not hasattr(TableDetails, arg):
+            raise ValueError(f"engine_{arg} is not a valid option")
+
     for p in params:
-        if p.required and p.name not in args:
+        if p.required and p.name not in engine_settings:
             raise ValueError(f"Missing required parameter '{p.name}'")
-        param_value = args.get(p.name, None) or p.default_value
+        param_value = engine_settings.get(p.name) or p.default_value
         if not param_value:
             continue
         if p.is_valid:
             check_is_valid(
-                valid_check=p.is_valid, check_type="parameter", columns=columns, tb_param=p.tb_param, value=param_value
+                valid_check=p.is_valid,
+                check_type="parameter",
+                columns=schema_columns,
+                tb_param=p.tb_param,
+                value=param_value,
             )
+
+    for o in options:
+        if o.required and o.name not in engine_settings:
+            raise ValueError(f"Missing required option '{o.name}'")
+        option_value = engine_settings.get(o.name) or o.default_value
+        if o.is_valid:
+            check_is_valid(
+                valid_check=o.is_valid,
+                check_type="option",
+                columns=schema_columns,
+                tb_param=o.tb_param,
+                value=option_value,
+            )
+
+
+def engine_params(params: List[EngineParam], args: Dict) -> List[str]:
+    params_values = []
+    for p in params:
+        param_value = args.get(p.name, None) or p.default_value
+        if not param_value:
+            continue
         params_values.append(param_value)
     return params_values
 
 
-def engine_options(columns, options: List[EngineOption], args: Dict):
+def engine_options(options: List[EngineOption], args: Dict) -> List[str]:
     options_values = []
     engine_settings = ""
 
     for o in options:
-        if o.required and o.name not in args:
-            raise ValueError(f"Missing required option '{o.name}'")
         option_value = args.get(o.name) or o.default_value
-        if o.is_valid:
-            check_is_valid(
-                valid_check=o.is_valid, check_type="option", columns=columns, tb_param=o.tb_param, value=option_value
-            )
-
         if option_value:
             if o.sql.lower() == "settings":
                 engine_settings = f"{o.sql} {option_value}"
+            # Conditional TTLs fail if we wrap the value in parentheses
+            elif o.sql.lower() == "ttl":
+                options_values.append(f"{o.sql} {option_value}")
             else:
                 options_values.append(f"{o.sql} ({option_value})")
 
@@ -491,7 +533,7 @@ def check_is_valid(
     columns: List[Dict[str, Any]],
     tb_param: str,
     value: str,
-):
+) -> None:
     """
     >>> check_is_valid(sorting_key_is_valid, 'option', ['column-name'], 'sorting_key', 'date')
 
@@ -508,15 +550,13 @@ def check_is_valid(
         raise ValueError(f"Invalid value '{value}' for {check_type} '{tb_param}', reason: {e}")
 
 
-def build_engine(
-    engine: str, columns: Optional[List], params: List[EngineParam], options: List[EngineOption], args: Dict
-):
-    return f"{engine}({', '.join(engine_params(columns, params, args))}) {' '.join(engine_options(columns, options, args))}".strip()
+def build_engine(engine: str, params: List[EngineParam], options: List[EngineOption], args: Dict) -> str:
+    return f"{engine}({', '.join(engine_params(params, args))}) {' '.join(engine_options(options, args))}".strip()
 
 
 def engine_full_from_dict(
     engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
-):
+) -> str:
     """
     >>> schema = ''
     >>> engine_full_from_dict('wadus', {}, schema=schema)
@@ -536,7 +576,9 @@ def engine_full_from_dict(
     >>> engine_full_from_dict('MergeTree', {'sorting_key': 'local_date, cod_store'}, schema=schema)
     'MergeTree() ORDER BY (local_date, cod_store)'
     >>> engine_full_from_dict('MergeTree', {'partition_key': 'toDate(timestamp)', 'sorting_key': 'local_date, cod_store', 'settings': 'index_granularity = 32, index_granularity_bytes = 2048', 'ttl': 'toDate(local_date) + INTERVAL 1 DAY'}, schema=schema)
-    'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL (toDate(local_date) + INTERVAL 1 DAY) SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
+    'MergeTree() PARTITION BY (toDate(timestamp)) ORDER BY (local_date, cod_store) TTL toDate(local_date) + INTERVAL 1 DAY SETTINGS index_granularity = 32, index_granularity_bytes = 2048'
+    >>> engine_full_from_dict('MergeTree', {'sorting_key': 'timestamp', 'ttl': 'timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'}, schema=schema)
+    'MergeTree() ORDER BY (timestamp) TTL timestamp + INTERVAL 1 DAY DELETE WHERE is_deleted = 1'
 
     >>> schema = ''
     >>> engine_full_from_dict('CollapsingMergeTree', {'sign': 'sign_column'}, schema=schema)
@@ -605,20 +647,10 @@ def engine_full_from_dict(
     'ReplacingMergeTree(updated_at) PARTITION BY (tuple()) ORDER BY (project_id) SETTINGS index_granularity = 32'
     """
 
-    if schema is not None and columns is not None:
-        raise ValueError("You can not use 'schema' and 'columns' at the same time")
-    engine_config = get_engine_config(engine)
-    name, (params, options) = engine_config
-    if columns is None and schema is not None:
-        columns = parse_table_structure(schema)
-
+    validate_engine_config(engine, args, schema=schema, columns=columns)
+    name, (params, options) = get_engine_config(engine)
     engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
-
-    for arg in engine_settings:
-        if not hasattr(TableDetails, arg):
-            raise ValueError(f"engine_{arg} is not a valid option")
-
-    return build_engine(name, columns, params, options, engine_settings)
+    return build_engine(name, params, options, engine_settings)
 
 
 def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:
@@ -694,8 +726,7 @@ def ttl_from_engine(engine: str) -> Optional[str]:
         return None
     settings_array = engine.split(" SETTINGS ")
     settings = " SETTINGS " + settings_array[1] if len(settings_array) > 1 else None
-    ttl = ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
-    return ttl
+    return ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
 
 
 def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:
@@ -759,17 +790,22 @@ def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:
         return None
 
 
+@functools.lru_cache(maxsize=1024)
+def _index_lines(create_table_query_expr: str) -> tuple[str, ...]:
+    from tinybird.sql_toolset import format_sql
+
+    return tuple(
+        line.strip() for line in format_sql(create_table_query_expr).splitlines() if line.strip().startswith("INDEX")
+    )
+
+
 def _parse_indexes(create_table_query_expr: str) -> List[TableIndex]:
-    if create_table_query_expr == "":
+    # format_sql is CPU-heavy and runs on the event loop. ClickHouse writes the
+    # INDEX keyword in uppercase in create_table_query.
+    if "INDEX" not in create_table_query_expr:
         return []
     try:
-        from tinybird.sql_toolset import format_sql
-
-        indexes = [
-            line.strip()
-            for line in format_sql(create_table_query_expr).splitlines()
-            if line.strip().startswith("INDEX")
-        ]
+        indexes = list(_index_lines(create_table_query_expr))
     except ModuleNotFoundError:
         # this is not needed from CLI
         return []