tinybird 4.6.18.dev0__tar.gz → 4.6.19__tar.gz

{tinybird-4.6.18.dev0 → tinybird-4.6.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tinybird
-Version: 4.6.18.dev0
+Version: 4.6.19
 Summary: Tinybird Command Line Tool
 Home-page: https://www.tinybird.co/docs/forward/commands
 Author: Tinybird
@@ -52,6 +52,26 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
 Changelog
 ----------
 
+4.6.19
+********
+
+- `Changed` ``tb datasource append`` and ``tb datasource replace`` submit remote CSV, NDJSON, and Parquet URLs through the v1 import endpoint by default.
+
+4.6.18
+********
+
+- `Fixed` CLI commands (for example ``tb migrate-to-forward``) no longer abort on startup when the best-effort refresh of the cached workspace id/name fails for a reason other than authentication.
+
+4.6.17
+********
+
+- `Added` ``tb job retry <job_id>`` to retry eligible import and S3/GCS sync jobs in ``error`` or ``cancelled`` status.
+
+4.6.18
+********
+
+- `Added` ``Markdown`` is now a valid ``FORMAT`` for SQL queries (for example ``tb sql "SELECT 1 FORMAT Markdown"``), returned as a Markdown table.
+
 4.6.16
 ********
 

tinybird-4.6.19/pyproject.toml

@@ -0,0 +1,228 @@
+[build-system]
+requires = [
+    "setuptools>=61.0",
+    "wheel==0.37.1",
+    "cffi==1.17.1",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tinybird"
+version = "1.0"
+description = "tinybird"
+readme = { text = "tinybird analytics", content-type = "text/markdown" }
+requires-python = ">=3.13,<3.15"
+authors = [
+    { name = "tinybird.co" },
+]
+dependencies = [
+    "GitPython==3.1.44",
+    "Markdown==3.5.1",
+    "aioboto3==14.0.0",
+    "aiofiles==24.1.0",
+    "aiohttp==3.11.2",
+    "anthropic==0.55.0",
+    "aws-msk-iam-sasl-signer-python==1.0.2",
+    "cachetools==5.5.0",
+    "cffi==1.17.1",
+    "chardet==4.0.0",
+    "ciso8601==2.2.0",
+    "click==8.1.8",
+    "clickhouse-cityhash==1.0.2.5",
+    "clickhouse-driver==0.2.9",
+    "confluent-kafka[avro,json,schemaregistry]==2.12.2",
+    "contextvars==2.4",
+    "cramjam==2.11.0",
+    "croniter==6.2.2",
+    "cryptography==3.4.8",
+    "datasketch==1.6.5",
+    "dnspython==2.8.0",
+    "docker==7.1.0",
+    "fastavro==1.12.2",
+    "fastmcp==2.12.2",
+    "google-cloud-aiplatform==1.148.1",
+    "google-cloud-compute==1.48.0",
+    "google-cloud-scheduler==2.20.0",
+    "google-cloud-secret-manager==2.29.0",
+    "google-cloud-storage==3.4.1",
+    "google_api_core==2.25.2",
+    "googleapis_common_protos==1.66.0",
+    "grpcio_status==1.67.1",
+    "humanfriendly==10.0",
+    "isal==1.8.0",
+    "jinja2",
+    "jsondiff==2.2.1",
+    "jsonschema==4.23.0",
+    "litellm==1.80.11",
+    "logfire-api==4.2.0",
+    "lru-dict==1.3.0",
+    "lz4==4.4.5",
+    "memray==1.20.0",
+    "msgspec==0.21.1",
+    "multidict==6.6.4",
+    "networkx==3.4.2",
+    "numpy==1.26.4",
+    "objgraph==3.6.2",
+    "opentelemetry-api==1.38.0",
+    "opentelemetry-exporter-otlp-proto-http==1.38.0",
+    "opentelemetry-sdk==1.38.0",
+    "orb-billing==2.17.0",
+    "orjson==3.11.5",
+    "packaging==23.2",
+    "passlib==1.7.1",
+    "posthog==7.28.0",
+    "pre-commit==4.6.2",
+    "prometheus_client==0.26.0",
+    "prompt_toolkit==3.0.48",
+    "propcache==0.3.2",
+    "proto_plus==1.25.0",
+    "protobuf==5.29.5",
+    "psutil==7.2.2",
+    "psycopg2-binary==2.9.12",
+    "py-spy==0.4.2",
+    "pyarrow==23.0.1",
+    "pycurl==7.45.7",
+    "pydantic-ai-slim[anthropic]==0.5.0",
+    "pydantic-ai-slim[google]==0.5.0",
+    "pydantic-ai-slim[retries]==0.5.0",
+    "pydantic==2.12.5",
+    "pyjwt[crypto]==2.9.0",
+    "pympler==1.1",
+    "pynacl==1.5.0",
+    "pyperclip==1.9.0",
+    "python-dateutil==2.9.0.post0",
+    "python-dotenv==1.1.0",
+    "python-ulid==2.7.0",
+    "pyyaml==6.0.1",
+    "redis[hiredis]==5.3.1",
+    "requests==2.32.4",
+    "sentry-sdk==2.69.1",
+    "setproctitle==1.3.3",
+    "shandy-sqlfmt==0.11.1",
+    "shandy-sqlfmt[jinjafmt]==0.11.1",
+    "sqlparse==0.4.3",
+    "starlette==1.0.1",
+    "streaming-form-data==1.20.0",
+    "stripe==3.1.0",
+    "tabulate==0.8.10",
+    "thefuzz==0.22.1",
+    "tinybird-python-sdk[ai]==0.3.7",
+    "tinybird-toolset==2.6.2",
+    "toposort==1.10",
+    "tornado==6.5.5",
+    "urllib3==1.26.20",
+    "uvicorn[standard]==0.49.0",
+    "virtualenv==20.21.1",
+    "watchdog==6.0.0",
+    "wheel==0.37.1",
+    "zlib-ng==1.0.0",
+    "zstandard==0.25.0",
+]
+
+[dependency-groups]
+deploy = [
+    "GitPython==3.1.44",
+    "ansible==8.5.0",
+    "ara==1.6.1",
+    "boto3==1.36.21",
+    "fabric==3.2.2",
+    "google-auth==2.47.0",
+    "jsonnet",
+    "prettytable==3.8.0",
+    "requests==2.32.4",
+    "requests[security]",
+    "xmltodict",
+]
+devtools = [
+    "flameprof",
+    "matplotlib",
+    "memory_profiler",
+    "rope",
+]
+doc = [
+    "Pallets-Sphinx-Themes==2.3.0",
+    "pyOpenSSL<24.0.0",
+    "sphinx-autobuild==2024.2.4",
+    "sphinx-sitemap",
+    "sphinx==7.3.7",
+    "sphinxcontrib-httpdomain",
+    "sphinxcontrib-jquery==4.1",
+]
+lint = [
+    "botocore-stubs==1.36.23",
+    "mypy==2.3.1",
+    "ruff==0.16.8",
+    "types-Markdown",
+    "types-PyYAML",
+    "types-aiofiles==24.1.0.20241221",
+    "types-boto3-lite==1.36.21",
+    "types-cachetools",
+    "types-chardet",
+    "types-paramiko",
+    "types-pycurl",
+    "types-python-dateutil",
+    "types-pytz",
+    "types-requests",
+    "types-tabulate",
+    "types_croniter",
+]
+test = [
+    "coverage==7.10.6",
+    "freezegun==1.5.5",
+    "google-auth==2.47.0",
+    "gsutil==4.58",
+    "hypothesis==6.156.1",
+    "oauth2client==3.0.0",
+    "py==1.11.0",
+    "pytest-asyncio==0.23.8",
+    "pytest-cov==4.1.0",
+    "pytest-env==1.1.3",
+    "pytest-flakefinder==1.1.0",
+    "pytest-timeout==2.3.1",
+    "pytest-xdist==3.6.1",
+    "pytest==8.1.2",
+    "rangehttpserver==1.4.0",
+    "yappi==1.7.6",
+]
+
+[project.scripts]
+tinybird_server = "tinybird.app:run"
+tinybird_tool = "tinybird.tinybird_tool.main:cli"
+tb_old = "tinybird.tb_cli:cli"
+tb = "tinybird.tb.cli:cli"
+colibri = "tinybird.colibri.cli:cli"
+
+[tool.uv.workspace]
+members = ["data_connector"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+namespaces = false
+include = ["tinybird", "tinybird.*", "tinybird_shared", "tinybird_shared.*", "scripts"]
+exclude = ["tests*"]
+
+[tool.setuptools.package-data]
+tinybird = [
+    "templates/**/*",
+    "static/**/*",
+    "default_pipes/**/*",
+    "sql/**/*",
+    "monitors/**/*.sql",
+    "scripts/**/*",
+    "data_projects/**/*",
+    "data_projects/**/.*",
+    "raw_events/**/*",
+    "query_booster/**/*",
+    "deployments/**/*",
+]
+tinybird_shared = [
+    "redis_client/**/*",
+    "retry/**/*",
+    "metrics/**/*",
+    "gatherer_settings/**/*",
+    "clickhouse/**/*",
+    "gatherer_backpressure/**/*",
+    "data_connector_circuit_breaker/**/*",
+]

{tinybird-4.6.18.dev0 → tinybird-4.6.19}/tinybird/ch_utils/constants.py

@@ -263,4 +263,5 @@ VALID_QUERY_FORMATS = (
     "JSONCompact",
     "JSONStringsEachRowWithProgress",
     "ODBCDriver2",
+    "Markdown",
 )

{tinybird-4.6.18.dev0 → tinybird-4.6.19}/tinybird/ch_utils/engine.py

@@ -111,8 +111,7 @@ class TableDetails:
 
     @property
     def version(self):
-        _version = self.details.get("version", None)
-        return _version
+        return self.details.get("version", None)
 
     def is_replicated(self) -> bool:
         engine: Optional[str] = self.details.get("engine", None)
@@ -183,6 +182,7 @@ class TableDetails:
         settings = self.details.get("settings", None)
         if settings and settings.strip().lower() != "index_granularity = 8192":
             return settings
+        return None
 
     @property
     def ttl(self):
@@ -190,23 +190,19 @@ class TableDetails:
 
     @property
     def ver(self):
-        _ver = self.details.get("ver", None)
-        return _ver
+        return self.details.get("ver", None)
 
     @property
     def is_deleted(self):
-        _is_deleted = self.details.get("is_deleted", None)
-        return _is_deleted
+        return self.details.get("is_deleted", None)
 
     @property
     def columns(self):
-        _columns = self.details.get("columns", None)
-        return _columns
+        return self.details.get("columns", None)
 
     @property
     def sign(self):
-        _sign = self.details.get("sign", None)
-        return _sign
+        return self.details.get("sign", None)
 
     @property
     def statistics(self) -> Dict[str, Any]:
@@ -444,35 +440,64 @@ def get_engine_config(engine: str):
     )
 
 
-def engine_params(columns, params: List[EngineParam], args: Dict):
-    params_values = []
+def validate_engine_config(
+    engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
+) -> None:
+    if schema is not None and columns is not None:
+        raise ValueError("You can not use 'schema' and 'columns' at the same time")
+    _, (params, options) = get_engine_config(engine)
+    schema_columns = parse_table_structure(schema) if schema is not None else (columns or [])
+    engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
+
+    for arg in engine_settings:
+        if not hasattr(TableDetails, arg):
+            raise ValueError(f"engine_{arg} is not a valid option")
+
     for p in params:
-        if p.required and p.name not in args:
+        if p.required and p.name not in engine_settings:
             raise ValueError(f"Missing required parameter '{p.name}'")
-        param_value = args.get(p.name, None) or p.default_value
+        param_value = engine_settings.get(p.name) or p.default_value
         if not param_value:
             continue
         if p.is_valid:
             check_is_valid(
-                valid_check=p.is_valid, check_type="parameter", columns=columns, tb_param=p.tb_param, value=param_value
+                valid_check=p.is_valid,
+                check_type="parameter",
+                columns=schema_columns,
+                tb_param=p.tb_param,
+                value=param_value,
             )
+
+    for o in options:
+        if o.required and o.name not in engine_settings:
+            raise ValueError(f"Missing required option '{o.name}'")
+        option_value = engine_settings.get(o.name) or o.default_value
+        if o.is_valid:
+            check_is_valid(
+                valid_check=o.is_valid,
+                check_type="option",
+                columns=schema_columns,
+                tb_param=o.tb_param,
+                value=option_value,
+            )
+
+
+def engine_params(params: List[EngineParam], args: Dict) -> List[str]:
+    params_values = []
+    for p in params:
+        param_value = args.get(p.name, None) or p.default_value
+        if not param_value:
+            continue
         params_values.append(param_value)
     return params_values
 
 
-def engine_options(columns, options: List[EngineOption], args: Dict):
+def engine_options(options: List[EngineOption], args: Dict) -> List[str]:
     options_values = []
     engine_settings = ""
 
     for o in options:
-        if o.required and o.name not in args:
-            raise ValueError(f"Missing required option '{o.name}'")
         option_value = args.get(o.name) or o.default_value
-        if o.is_valid:
-            check_is_valid(
-                valid_check=o.is_valid, check_type="option", columns=columns, tb_param=o.tb_param, value=option_value
-            )
-
         if option_value:
             if o.sql.lower() == "settings":
                 engine_settings = f"{o.sql} {option_value}"
@@ -494,7 +519,7 @@ def check_is_valid(
     columns: List[Dict[str, Any]],
     tb_param: str,
     value: str,
-):
+) -> None:
     """
     >>> check_is_valid(sorting_key_is_valid, 'option', ['column-name'], 'sorting_key', 'date')
 
@@ -511,15 +536,13 @@ def check_is_valid(
         raise ValueError(f"Invalid value '{value}' for {check_type} '{tb_param}', reason: {e}")
 
 
-def build_engine(
-    engine: str, columns: Optional[List], params: List[EngineParam], options: List[EngineOption], args: Dict
-):
-    return f"{engine}({', '.join(engine_params(columns, params, args))}) {' '.join(engine_options(columns, options, args))}".strip()
+def build_engine(engine: str, params: List[EngineParam], options: List[EngineOption], args: Dict) -> str:
+    return f"{engine}({', '.join(engine_params(params, args))}) {' '.join(engine_options(options, args))}".strip()
 
 
 def engine_full_from_dict(
     engine: str, args: dict, schema: Optional[str] = None, columns: Optional[List[Dict[str, Any]]] = None
-):
+) -> str:
     """
     >>> schema = ''
     >>> engine_full_from_dict('wadus', {}, schema=schema)
@@ -610,20 +633,10 @@ def engine_full_from_dict(
     'ReplacingMergeTree(updated_at) PARTITION BY (tuple()) ORDER BY (project_id) SETTINGS index_granularity = 32'
     """
 
-    if schema is not None and columns is not None:
-        raise ValueError("You can not use 'schema' and 'columns' at the same time")
-    engine_config = get_engine_config(engine)
-    name, (params, options) = engine_config
-    if columns is None and schema is not None:
-        columns = parse_table_structure(schema)
-
+    validate_engine_config(engine, args, schema=schema, columns=columns)
+    name, (params, options) = get_engine_config(engine)
     engine_settings = {key.replace("engine_", ""): value for key, value in args.items()}
-
-    for arg in engine_settings:
-        if not hasattr(TableDetails, arg):
-            raise ValueError(f"engine_{arg} is not a valid option")
-
-    return build_engine(name, columns, params, options, engine_settings)
+    return build_engine(name, params, options, engine_settings)
 
 
 def engine_params_from_engine_full(engine_full: str) -> Dict[str, Any]:
@@ -699,8 +712,7 @@ def ttl_from_engine(engine: str) -> Optional[str]:
         return None
     settings_array = engine.split(" SETTINGS ")
     settings = " SETTINGS " + settings_array[1] if len(settings_array) > 1 else None
-    ttl = ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
-    return ttl
+    return ttl_array[1][: -(len(settings))] if settings else ttl_array[1]
 
 
 def ttl_condition_from_engine_full(engine_full: Optional[str]) -> Optional[str]:

{tinybird-4.6.18.dev0 → tinybird-4.6.19}/tinybird/client.py

@@ -266,14 +266,13 @@ class TinyB:
 
     async def alter_tokens(self, name: str, scopes: List[str]):
         if not scopes:
-            return
+            return None
         scopes_url: str = "&".join([f"scope={scope}" for scope in scopes])
         url = f"/v0/tokens/{name}"
         if len(url + "?" + scopes_url) > TinyB.MAX_GET_LENGTH:
             return await self._req(url, method="PUT", data=scopes_url)
-        else:
-            url = url + "?" + scopes_url
-            return await self._req(url, method="PUT", data="")
+        url = url + "?" + scopes_url
+        return await self._req(url, method="PUT", data="")
 
     async def datasources(self, branch: Optional[str] = None, used_by: bool = False) -> List[Dict[str, Any]]:
         params = {}
@@ -294,16 +293,13 @@ class TinyB:
         return await self._req(f"/v0/variables/{name}")
 
     async def create_secret(self, name: str, value: str):
-        response = await self._req("/v0/variables", method="POST", data={"name": name, "value": value})
-        return response
+        return await self._req("/v0/variables", method="POST", data={"name": name, "value": value})
 
     async def update_secret(self, name: str, value: str):
-        response = await self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
-        return response
+        return await self._req(f"/v0/variables/{name}", method="PUT", data={"value": value})
 
     async def delete_secret(self, name: str):
-        response = await self._req(f"/v0/variables/{name}", method="DELETE")
-        return response
+        return await self._req(f"/v0/variables/{name}", method="DELETE")
 
     async def get_connections(self, service: Optional[str] = None):
         params = {}
@@ -412,6 +408,16 @@ class TinyB:
         format: str = "csv",
         replace_options: Optional[Set[str]] = None,
     ) -> Dict[str, Any]:
+        if mode in ("append", "replace"):
+            return await self.datasource_append_url(
+                table_name,
+                url,
+                mode=mode,
+                sql_condition=sql_condition,
+                format=format,
+                replace_options=replace_options,
+            )
+
         params = {"name": table_name, "url": url, "mode": mode, "debug": "blocks_block_log", "format": format}
 
         if sql_condition:
@@ -428,6 +434,33 @@ class TinyB:
 
         return await self.wait_for_job(res["id"], status_callback, backoff_multiplier=1.5, maximum_backoff_seconds=20)
 
+    async def datasource_append_url(
+        self,
+        datasource_name: str,
+        url: str,
+        mode: str = "append",
+        sql_condition: Optional[str] = None,
+        format: str = "csv",
+        replace_options: Optional[Set[str]] = None,
+    ) -> Dict[str, Any]:
+        params = {"format": format, "url": url}
+        if sql_condition:
+            params["replace_condition"] = sql_condition
+        if replace_options:
+            for option in replace_options:
+                params[option] = "true"
+
+        res = await self._req(
+            f"/v1/datasources/{quote(datasource_name, safe='')}/{mode}?{urlencode(params, safe='')}",
+            method="POST",
+            data=b"",
+        )
+        if not isinstance(res, dict):
+            raise RuntimeError("We couldn't confirm that your import started. Please try again.")
+        if "error" in res:
+            raise Exception(res["error"])
+        return res
+
     async def datasource_delete(self, datasource_name: str, force: bool = False, dry_run: bool = False):
         params = {"force": "true" if force else "false", "dry_run": "true" if dry_run else "false"}
         return await self._req(f"/v0/datasources/{datasource_name}?{urlencode(params)}", method="DELETE")
@@ -577,8 +610,7 @@ class TinyB:
         node_name = node["params"]["name"] if node.get("params", None) else node["name"]
         if datasource_name:
             params["datasource"] = datasource_name
-        response = await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
-        return response
+        return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/analysis?{urlencode(params)}")
 
     async def populate_node(
         self,
@@ -600,10 +632,7 @@ class TinyB:
             params.update({"populate_condition": populate_condition})
         if on_demand_compute:
             params.update({"on_demand_compute": "true"})
-        response = await self._req(
-            f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST"
-        )
-        return response
+        return await self._req(f"/v0/pipes/{pipe_name}/nodes/{node_name}/population?{urlencode(params)}", method="POST")
 
     async def pipes(self, branch=None, dependencies: bool = False, node_attrs=None, attrs=None) -> List[Dict[str, Any]]:
         params = {
@@ -635,9 +664,8 @@ class TinyB:
         query_string = urlencode(params)
         if len(url + "?" + query_string) > TinyB.MAX_GET_LENGTH:
             return await self._req(f"/v0/pipes/{pipe_name_or_uid}.{format}", method="POST", data=params)
-        else:
-            url = url + "?" + query_string
-            return await self._req(url)
+        url = url + "?" + query_string
+        return await self._req(url)
 
     async def pipe_create(self, pipe_name: str, sql: str):
         return await self._req(
@@ -726,8 +754,7 @@ class TinyB:
 
         if len(sql) > TinyB.MAX_GET_LENGTH:
             return await self._req(f"/v0/sql?{urlencode(params)}", data=sql, method="POST")
-        else:
-            return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
+        return await self._req(f"/v0/sql?q={quote(sql, safe='')}&{urlencode(params)}")
 
     async def jobs(self, status=None):
         jobs = (await self._req("/v0/jobs"))["jobs"]
@@ -922,7 +949,7 @@ class TinyB:
 
     async def add_workspaces_to_organization(self, organization_id: str, workspace_ids: List[str]):
         if not workspace_ids:
-            return
+            return None
         return await self._req(
             f"/v0/organizations/{organization_id}/workspaces",
             method="PUT",
@@ -1105,8 +1132,7 @@ class TinyB:
                 from tinybird.sql_toolset import format_sql
 
                 return format_sql(sql)
-            else:
-                return await self._sql_get_format_remote(sql, with_clickhouse_format)
+            return await self._sql_get_format_remote(sql, with_clickhouse_format)
         except ModuleNotFoundError:
             return await self._sql_get_format_remote(sql, with_clickhouse_format)
 
@@ -1168,8 +1194,7 @@ class TinyB:
         return next((connector for connector in result["connectors"] if connector_equals(connector, kwargs)), None)
 
     async def regions(self):
-        regions = await self._req("/v0/regions")
-        return regions
+        return await self._req("/v0/regions")
 
     async def datasource_query_copy(self, datasource_name: str, sql_query: str):
         params = {"copy_to": datasource_name}