tinybird-cli 5.22.3.dev0__py3-none-any.whl → 6.0.2.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tinybird/__cli__.py +2 -2
- tinybird/ch_utils/engine.py +2 -77
- tinybird/client.py +3 -141
- tinybird/config.py +0 -5
- tinybird/datafile_common.py +18 -38
- tinybird/datatypes.py +13 -0
- tinybird/feedback_manager.py +1 -41
- tinybird/sql.py +6 -5
- tinybird/sql_template.py +527 -36
- tinybird/sql_toolset.py +6 -32
- tinybird/tb_cli_modules/auth.py +1 -11
- tinybird/tb_cli_modules/cli.py +1 -83
- tinybird/tb_cli_modules/common.py +27 -153
- tinybird/tb_cli_modules/config.py +1 -3
- tinybird/tb_cli_modules/connection.py +1 -225
- tinybird/tb_cli_modules/datasource.py +11 -95
- tinybird/tornado_template.py +8 -3
- {tinybird_cli-5.22.3.dev0.dist-info → tinybird_cli-6.0.2.dev0.dist-info}/METADATA +13 -20
- tinybird_cli-6.0.2.dev0.dist-info/RECORD +44 -0
- tinybird/connectors.py +0 -422
- tinybird_cli-5.22.3.dev0.dist-info/RECORD +0 -45
- {tinybird_cli-5.22.3.dev0.dist-info → tinybird_cli-6.0.2.dev0.dist-info}/WHEEL +0 -0
- {tinybird_cli-5.22.3.dev0.dist-info → tinybird_cli-6.0.2.dev0.dist-info}/entry_points.txt +0 -0
- {tinybird_cli-5.22.3.dev0.dist-info → tinybird_cli-6.0.2.dev0.dist-info}/top_level.txt +0 -0
|
@@ -13,7 +13,6 @@ import click
|
|
|
13
13
|
from click import Context
|
|
14
14
|
|
|
15
15
|
from tinybird.client import DoesNotExistException, TinyB
|
|
16
|
-
from tinybird.config import FeatureFlags
|
|
17
16
|
from tinybird.feedback_manager import FeedbackManager
|
|
18
17
|
from tinybird.tb_cli_modules.cli import cli
|
|
19
18
|
from tinybird.tb_cli_modules.common import (
|
|
@@ -35,7 +34,6 @@ from tinybird.tb_cli_modules.common import (
|
|
|
35
34
|
validate_string_connector_param,
|
|
36
35
|
)
|
|
37
36
|
from tinybird.tb_cli_modules.exceptions import CLIConnectionException
|
|
38
|
-
from tinybird.tb_cli_modules.telemetry import is_ci_environment
|
|
39
37
|
|
|
40
38
|
DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
41
39
|
DataConnectorType.KAFKA: [
|
|
@@ -50,17 +48,6 @@ DATA_CONNECTOR_SETTINGS: Dict[DataConnectorType, List[str]] = {
|
|
|
50
48
|
"kafka_ssl_ca_pem",
|
|
51
49
|
],
|
|
52
50
|
DataConnectorType.GCLOUD_SCHEDULER: ["gcscheduler_region"],
|
|
53
|
-
DataConnectorType.SNOWFLAKE: [
|
|
54
|
-
"account",
|
|
55
|
-
"username",
|
|
56
|
-
"password",
|
|
57
|
-
"role",
|
|
58
|
-
"warehouse",
|
|
59
|
-
"warehouse_size",
|
|
60
|
-
"stage",
|
|
61
|
-
"integration",
|
|
62
|
-
],
|
|
63
|
-
DataConnectorType.BIGQUERY: ["account"],
|
|
64
51
|
DataConnectorType.GCLOUD_STORAGE: [
|
|
65
52
|
"gcs_private_key_id",
|
|
66
53
|
"gcs_client_x509_cert_url",
|
|
@@ -201,217 +188,6 @@ async def connection_create_kafka(
|
|
|
201
188
|
click.echo(FeedbackManager.success_connection_created(id=id))
|
|
202
189
|
|
|
203
190
|
|
|
204
|
-
@connection_create.command(
|
|
205
|
-
name="snowflake", short_help="Creates a Snowflake connection in the current workspace", hidden=True
|
|
206
|
-
)
|
|
207
|
-
@click.option("--account", help="The account identifier of your Snowflake account (e.g. myorg-account123)")
|
|
208
|
-
@click.option("--username", help="The Snowflake user you want to use for the connection")
|
|
209
|
-
@click.option("--password", help="The Snowflake password of the chosen user")
|
|
210
|
-
@click.option(
|
|
211
|
-
"--warehouse",
|
|
212
|
-
default=None,
|
|
213
|
-
help="If not provided, it's set to your Snowflake user default. Warehouse to run the export sentences.",
|
|
214
|
-
)
|
|
215
|
-
@click.option(
|
|
216
|
-
"--role",
|
|
217
|
-
default=None,
|
|
218
|
-
help="If not provided, it's set to your Snowflake user default. Snowflake role use in the export process.",
|
|
219
|
-
)
|
|
220
|
-
@click.option(
|
|
221
|
-
"--connection-name",
|
|
222
|
-
default=None,
|
|
223
|
-
help="The name of your Snowflake connection. If not provided, it's set as the account identifier",
|
|
224
|
-
)
|
|
225
|
-
@click.option(
|
|
226
|
-
"--integration-name",
|
|
227
|
-
default=None,
|
|
228
|
-
help="The name of your Snowflake integration. If not provided, we will create one.",
|
|
229
|
-
)
|
|
230
|
-
@click.option(
|
|
231
|
-
"--stage-name", default=None, help="The name of your Snowflake stage. If not provided, we will create one."
|
|
232
|
-
)
|
|
233
|
-
@click.option("--no-validate", is_flag=True, help="Do not validate Snowflake permissions during connection creation")
|
|
234
|
-
@click.pass_context
|
|
235
|
-
@coro
|
|
236
|
-
async def connection_create_snowflake(
|
|
237
|
-
ctx: Context,
|
|
238
|
-
account: Optional[str],
|
|
239
|
-
username: Optional[str],
|
|
240
|
-
password: Optional[str],
|
|
241
|
-
warehouse: Optional[str],
|
|
242
|
-
role: Optional[str],
|
|
243
|
-
connection_name: Optional[str],
|
|
244
|
-
integration_name: Optional[str],
|
|
245
|
-
stage_name: Optional[str],
|
|
246
|
-
no_validate: Optional[bool],
|
|
247
|
-
) -> None:
|
|
248
|
-
"""
|
|
249
|
-
Creates a Snowflake connection in the current workspace
|
|
250
|
-
|
|
251
|
-
\b
|
|
252
|
-
$ tb connection create snowflake
|
|
253
|
-
"""
|
|
254
|
-
|
|
255
|
-
snowflake_connector_enabled = FeatureFlags.enable_snowflake_connector_command()
|
|
256
|
-
if not snowflake_connector_enabled:
|
|
257
|
-
click.echo(FeedbackManager.error_snowflake_connector_not_enabled())
|
|
258
|
-
return
|
|
259
|
-
|
|
260
|
-
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
261
|
-
client: TinyB = obj["client"]
|
|
262
|
-
|
|
263
|
-
is_connection_valid: bool = True
|
|
264
|
-
|
|
265
|
-
if not username:
|
|
266
|
-
username = click.prompt("User (must have created stage and create integration in Snowflake)")
|
|
267
|
-
assert isinstance(username, str)
|
|
268
|
-
|
|
269
|
-
if not password:
|
|
270
|
-
password = click.prompt("Password", hide_input=True)
|
|
271
|
-
assert isinstance(password, str)
|
|
272
|
-
|
|
273
|
-
if not account:
|
|
274
|
-
account = click.prompt("Account identifier")
|
|
275
|
-
assert isinstance(account, str)
|
|
276
|
-
|
|
277
|
-
account_parts = account.split(".", maxsplit=1)
|
|
278
|
-
if len(account_parts) == 2:
|
|
279
|
-
account = "-".join(account_parts)
|
|
280
|
-
|
|
281
|
-
if not role:
|
|
282
|
-
roles = await client.get_snowflake_roles(account, username, password) or []
|
|
283
|
-
default_role = roles[0] if len(roles) else ""
|
|
284
|
-
role = click.prompt(
|
|
285
|
-
"Role (optional)",
|
|
286
|
-
type=click.types.Choice(roles, case_sensitive=False),
|
|
287
|
-
show_choices=True,
|
|
288
|
-
default=default_role,
|
|
289
|
-
show_default=True,
|
|
290
|
-
)
|
|
291
|
-
assert isinstance(role, str)
|
|
292
|
-
|
|
293
|
-
if not warehouse:
|
|
294
|
-
warehouses = await client.get_snowflake_warehouses(account, username, password, role) or []
|
|
295
|
-
warehouses_names = [w["name"] for w in warehouses]
|
|
296
|
-
default_warehouse = warehouses_names[0] if warehouses_names else ""
|
|
297
|
-
warehouse = click.prompt(
|
|
298
|
-
"Warehouse (optional)",
|
|
299
|
-
type=click.types.Choice(warehouses_names, case_sensitive=False),
|
|
300
|
-
default=default_warehouse,
|
|
301
|
-
show_default=False,
|
|
302
|
-
)
|
|
303
|
-
assert isinstance(warehouse, str)
|
|
304
|
-
|
|
305
|
-
if connection_name and no_validate is False:
|
|
306
|
-
if await client.get_connector(connection_name, "snowflake") is not None:
|
|
307
|
-
raise CLIConnectionException(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
308
|
-
else:
|
|
309
|
-
while not connection_name:
|
|
310
|
-
connection_name = click.prompt(
|
|
311
|
-
f"Connection name (optional, current: {account})", default=account, show_default=False
|
|
312
|
-
)
|
|
313
|
-
assert isinstance(connection_name, str)
|
|
314
|
-
|
|
315
|
-
if no_validate is False and await client.get_connector(connection_name, "snowflake") is not None:
|
|
316
|
-
click.echo(FeedbackManager.info_connection_already_exists(name=connection_name))
|
|
317
|
-
connection_name = None
|
|
318
|
-
assert isinstance(connection_name, str)
|
|
319
|
-
|
|
320
|
-
show_instructions: bool = not is_ci_environment()
|
|
321
|
-
|
|
322
|
-
if show_instructions:
|
|
323
|
-
instructions = await client.get_snowflake_integration_query(role, stage_name, integration_name)
|
|
324
|
-
if instructions:
|
|
325
|
-
for step in instructions.get("steps", []):
|
|
326
|
-
click.echo(step.get("description"))
|
|
327
|
-
click.echo("\n------")
|
|
328
|
-
click.echo(step.get("action"))
|
|
329
|
-
click.echo("------\n")
|
|
330
|
-
|
|
331
|
-
while True:
|
|
332
|
-
ans: str = click.prompt(
|
|
333
|
-
"Ready?", type=click.types.Choice(["Y", "n"], case_sensitive=False), default="Y", show_default=True
|
|
334
|
-
)
|
|
335
|
-
if ans.lower() == "y":
|
|
336
|
-
break
|
|
337
|
-
|
|
338
|
-
conn_file_name = f"{connection_name}.connection"
|
|
339
|
-
conn_file_path = Path(getcwd(), conn_file_name)
|
|
340
|
-
if os.path.isfile(conn_file_path):
|
|
341
|
-
raise CLIConnectionException(FeedbackManager.error_connection_file_already_exists(name=conn_file_name))
|
|
342
|
-
|
|
343
|
-
if no_validate is False:
|
|
344
|
-
click.echo("** Validating connection...")
|
|
345
|
-
is_connection_valid = await client.validate_snowflake_connection(account, username, password)
|
|
346
|
-
|
|
347
|
-
if not is_connection_valid:
|
|
348
|
-
raise CLIConnectionException(FeedbackManager.error_snowflake_improper_permissions())
|
|
349
|
-
|
|
350
|
-
_ = await client.connection_create_snowflake(
|
|
351
|
-
account, username, password, warehouse, role, connection_name, integration_name, stage_name
|
|
352
|
-
)
|
|
353
|
-
|
|
354
|
-
async with aiofiles.open(conn_file_path, "w") as f:
|
|
355
|
-
await f.write(
|
|
356
|
-
f"""TYPE snowflake
|
|
357
|
-
|
|
358
|
-
USERNAME='{username}'
|
|
359
|
-
ACCOUNT='{account}'
|
|
360
|
-
WAREHOUSE='{warehouse}'
|
|
361
|
-
ROLE='{role}'
|
|
362
|
-
"""
|
|
363
|
-
)
|
|
364
|
-
click.echo(FeedbackManager.success_connection_file_created(name=conn_file_name))
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
@connection_create.command(name="bigquery", short_help="Add a BigQuery connection", hidden="True")
|
|
368
|
-
@click.option("--no-validate", is_flag=True, help="Do not validate GCP permissions during connection creation")
|
|
369
|
-
@click.pass_context
|
|
370
|
-
@coro
|
|
371
|
-
async def connection_create_bigquery(ctx: Context, no_validate: bool) -> None:
|
|
372
|
-
"""
|
|
373
|
-
Add a BigQuery connection
|
|
374
|
-
|
|
375
|
-
\b
|
|
376
|
-
$ tb connection create bigquery
|
|
377
|
-
"""
|
|
378
|
-
|
|
379
|
-
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
380
|
-
client: TinyB = obj["client"]
|
|
381
|
-
|
|
382
|
-
click.echo(FeedbackManager.warning_bigquery_connector_deprecated())
|
|
383
|
-
click.echo("\n")
|
|
384
|
-
|
|
385
|
-
gcp_account_details: Dict[str, Any] = await client.get_gcp_service_account_details()
|
|
386
|
-
|
|
387
|
-
connection_created: bool = False
|
|
388
|
-
|
|
389
|
-
while True:
|
|
390
|
-
response = click.prompt(
|
|
391
|
-
FeedbackManager.prompt_bigquery_account(service_account=gcp_account_details["account"]),
|
|
392
|
-
type=click.Choice(["y", "N"], case_sensitive=False),
|
|
393
|
-
default="N",
|
|
394
|
-
show_default=True,
|
|
395
|
-
show_choices=True,
|
|
396
|
-
)
|
|
397
|
-
|
|
398
|
-
if response in ("n", "N"):
|
|
399
|
-
click.echo(FeedbackManager.info_cancelled_by_user())
|
|
400
|
-
break
|
|
401
|
-
|
|
402
|
-
if no_validate or await client.check_gcp_read_permissions():
|
|
403
|
-
connection_created = True
|
|
404
|
-
break
|
|
405
|
-
else:
|
|
406
|
-
click.echo("\n")
|
|
407
|
-
click.echo(FeedbackManager.error_bigquery_improper_permissions())
|
|
408
|
-
|
|
409
|
-
if connection_created:
|
|
410
|
-
async with aiofiles.open(Path(getcwd(), "bigquery.connection"), "w") as f:
|
|
411
|
-
await f.write("TYPE bigquery\n")
|
|
412
|
-
click.echo(FeedbackManager.success_connection_created(id="bigquery"))
|
|
413
|
-
|
|
414
|
-
|
|
415
191
|
@connection.command(name="rm")
|
|
416
192
|
@click.argument("connection_id_or_name")
|
|
417
193
|
@click.option(
|
|
@@ -636,7 +412,7 @@ async def connection_create_gcs_hmac(
|
|
|
636
412
|
@click.option("--private-key", help="Your GCS private key with access to the buckets")
|
|
637
413
|
@click.option("--private-key-id", help="Your GCS private key id with access to the buckets")
|
|
638
414
|
@click.option("--connection-name", default=None, help="The name of the connection to identify it in Tinybird")
|
|
639
|
-
@click.option("--no-validate", is_flag=True, help="Do not validate
|
|
415
|
+
@click.option("--no-validate", is_flag=True, help="Do not validate GCS permissions during connection creation")
|
|
640
416
|
@click.pass_context
|
|
641
417
|
@coro
|
|
642
418
|
async def connection_create_gcs(
|
|
@@ -7,7 +7,7 @@ import asyncio
|
|
|
7
7
|
import json
|
|
8
8
|
import logging
|
|
9
9
|
import re
|
|
10
|
-
from typing import
|
|
10
|
+
from typing import Any, Dict, List, Optional
|
|
11
11
|
|
|
12
12
|
import click
|
|
13
13
|
import humanfriendly
|
|
@@ -15,11 +15,6 @@ from click import Context
|
|
|
15
15
|
|
|
16
16
|
from tinybird.client import AuthNoTokenException, CanNotBeDeletedException, DoesNotExistException, TinyB
|
|
17
17
|
from tinybird.config import get_display_host
|
|
18
|
-
from tinybird.tb_cli_modules.config import CLIConfig
|
|
19
|
-
|
|
20
|
-
if TYPE_CHECKING:
|
|
21
|
-
from tinybird.connectors import Connector
|
|
22
|
-
|
|
23
18
|
from tinybird.datafile_common import get_name_version, wait_job
|
|
24
19
|
from tinybird.feedback_manager import FeedbackManager
|
|
25
20
|
from tinybird.tb_cli_modules.branch import warn_if_in_live
|
|
@@ -33,7 +28,6 @@ from tinybird.tb_cli_modules.common import (
|
|
|
33
28
|
coro,
|
|
34
29
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
35
30
|
get_format_from_filename_or_url,
|
|
36
|
-
load_connector_config,
|
|
37
31
|
push_data,
|
|
38
32
|
sync_data,
|
|
39
33
|
validate_datasource_name,
|
|
@@ -41,6 +35,7 @@ from tinybird.tb_cli_modules.common import (
|
|
|
41
35
|
validate_kafka_group,
|
|
42
36
|
validate_kafka_topic,
|
|
43
37
|
)
|
|
38
|
+
from tinybird.tb_cli_modules.config import CLIConfig
|
|
44
39
|
from tinybird.tb_cli_modules.exceptions import CLIDatasourceException
|
|
45
40
|
|
|
46
41
|
|
|
@@ -125,12 +120,6 @@ async def datasource_ls(ctx: Context, match: Optional[str], format_: str):
|
|
|
125
120
|
@datasource.command(name="append")
|
|
126
121
|
@click.argument("datasource_name")
|
|
127
122
|
@click.argument("url", nargs=-1)
|
|
128
|
-
@click.option(
|
|
129
|
-
"--connector",
|
|
130
|
-
type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
|
|
131
|
-
help="Import from one of the selected connectors",
|
|
132
|
-
hidden=True,
|
|
133
|
-
)
|
|
134
123
|
@click.option("--sql", default=None, help="Query to extract data from one of the SQL connectors", hidden=True)
|
|
135
124
|
@click.option(
|
|
136
125
|
"--incremental",
|
|
@@ -152,7 +141,6 @@ async def datasource_append(
|
|
|
152
141
|
ctx: Context,
|
|
153
142
|
datasource_name: str,
|
|
154
143
|
url,
|
|
155
|
-
connector: Optional[str],
|
|
156
144
|
sql: Optional[str],
|
|
157
145
|
incremental: Optional[str],
|
|
158
146
|
ignore_empty: bool,
|
|
@@ -165,62 +153,26 @@ async def datasource_append(
|
|
|
165
153
|
|
|
166
154
|
- Load from local file `tb datasource append [datasource_name] /path/to/local/file`
|
|
167
155
|
|
|
168
|
-
- Load from connector `tb datasource append [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from]`
|
|
169
156
|
"""
|
|
170
157
|
|
|
171
|
-
if not url
|
|
172
|
-
raise CLIDatasourceException(FeedbackManager.
|
|
173
|
-
|
|
174
|
-
if incremental and not connector:
|
|
175
|
-
raise CLIDatasourceException(FeedbackManager.error_incremental_not_supported())
|
|
176
|
-
|
|
177
|
-
if incremental:
|
|
178
|
-
date = None
|
|
179
|
-
source_column = incremental.split(":")[0]
|
|
180
|
-
dest_column = incremental.split(":")[-1]
|
|
181
|
-
client: TinyB = ctx.obj["client"]
|
|
182
|
-
result = await client.query(f"SELECT max({dest_column}) as inc from {datasource_name} FORMAT JSON")
|
|
183
|
-
try:
|
|
184
|
-
date = result["data"][0]["inc"]
|
|
185
|
-
except Exception as e:
|
|
186
|
-
raise CLIDatasourceException(f"{str(e)}")
|
|
187
|
-
if date:
|
|
188
|
-
sql = f"{sql} WHERE {source_column} > '{date}'"
|
|
189
|
-
await push_data(
|
|
190
|
-
ctx, datasource_name, url, connector, sql, mode="append", ignore_empty=ignore_empty, concurrency=concurrency
|
|
191
|
-
)
|
|
158
|
+
if not url:
|
|
159
|
+
raise CLIDatasourceException(FeedbackManager.error_missing_url(datasource=datasource_name))
|
|
160
|
+
await push_data(ctx, datasource_name, url, mode="append", concurrency=concurrency)
|
|
192
161
|
|
|
193
162
|
|
|
194
163
|
@datasource.command(name="replace")
|
|
195
164
|
@click.argument("datasource_name")
|
|
196
165
|
@click.argument("url", nargs=-1)
|
|
197
|
-
@click.option(
|
|
198
|
-
"--connector",
|
|
199
|
-
type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
|
|
200
|
-
help="Import from one of the selected connectors",
|
|
201
|
-
hidden=True,
|
|
202
|
-
)
|
|
203
|
-
@click.option("--sql", default=None, help="Query to extract data from one of the SQL connectors", hidden=True)
|
|
204
166
|
@click.option("--sql-condition", default=None, help="SQL WHERE condition to replace data", hidden=True)
|
|
205
167
|
@click.option("--skip-incompatible-partition-key", is_flag=True, default=False, hidden=True)
|
|
206
|
-
@click.option(
|
|
207
|
-
"--ignore-empty",
|
|
208
|
-
help="Wheter or not to ignore empty results from the connector",
|
|
209
|
-
is_flag=True,
|
|
210
|
-
default=False,
|
|
211
|
-
hidden=True,
|
|
212
|
-
)
|
|
213
168
|
@click.pass_context
|
|
214
169
|
@coro
|
|
215
170
|
async def datasource_replace(
|
|
216
171
|
ctx,
|
|
217
172
|
datasource_name,
|
|
218
173
|
url,
|
|
219
|
-
connector,
|
|
220
|
-
sql,
|
|
221
174
|
sql_condition,
|
|
222
175
|
skip_incompatible_partition_key,
|
|
223
|
-
ignore_empty: bool,
|
|
224
176
|
):
|
|
225
177
|
"""
|
|
226
178
|
Replaces the data in a data source from a URL, local file or a connector
|
|
@@ -229,11 +181,10 @@ async def datasource_replace(
|
|
|
229
181
|
|
|
230
182
|
- Replace from local file `tb datasource replace [datasource_name] /path/to/local/file --sql-condition "country='ES'"`
|
|
231
183
|
|
|
232
|
-
- Replace from connector `tb datasource replace [datasource_name] --connector [connector_name] --sql [the_sql_to_extract_from] --sql-condition "country='ES'"`
|
|
233
184
|
"""
|
|
234
185
|
|
|
235
|
-
if not url
|
|
236
|
-
raise CLIDatasourceException(FeedbackManager.
|
|
186
|
+
if not url:
|
|
187
|
+
raise CLIDatasourceException(FeedbackManager.error_missing_url(datasource=datasource_name))
|
|
237
188
|
|
|
238
189
|
replace_options = set()
|
|
239
190
|
if skip_incompatible_partition_key:
|
|
@@ -242,37 +193,20 @@ async def datasource_replace(
|
|
|
242
193
|
ctx,
|
|
243
194
|
datasource_name,
|
|
244
195
|
url,
|
|
245
|
-
connector,
|
|
246
|
-
sql,
|
|
247
196
|
mode="replace",
|
|
248
197
|
sql_condition=sql_condition,
|
|
249
198
|
replace_options=replace_options,
|
|
250
|
-
ignore_empty=ignore_empty,
|
|
251
199
|
)
|
|
252
200
|
|
|
253
201
|
|
|
254
202
|
@datasource.command(name="analyze")
|
|
255
203
|
@click.argument("url_or_file")
|
|
256
|
-
@click.option(
|
|
257
|
-
"--connector",
|
|
258
|
-
type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
|
|
259
|
-
help="Use from one of the selected connectors. In this case pass a table name as a parameter instead of a file name or an URL",
|
|
260
|
-
hidden=True,
|
|
261
|
-
)
|
|
262
204
|
@click.pass_context
|
|
263
205
|
@coro
|
|
264
|
-
async def datasource_analyze(ctx, url_or_file
|
|
206
|
+
async def datasource_analyze(ctx, url_or_file):
|
|
265
207
|
"""Analyze a URL or a file before creating a new data source"""
|
|
266
208
|
client = ctx.obj["client"]
|
|
267
209
|
|
|
268
|
-
_connector = None
|
|
269
|
-
if connector:
|
|
270
|
-
load_connector_config(ctx, connector, False, check_uninstalled=False)
|
|
271
|
-
if connector not in ctx.obj:
|
|
272
|
-
raise CLIDatasourceException(FeedbackManager.error_connector_not_configured(connector=connector))
|
|
273
|
-
else:
|
|
274
|
-
_connector = ctx.obj[connector]
|
|
275
|
-
|
|
276
210
|
def _table(title, columns, data):
|
|
277
211
|
row_format = "{:<25}" * len(columns)
|
|
278
212
|
click.echo(FeedbackManager.info_datasource_title(title=title))
|
|
@@ -280,9 +214,7 @@ async def datasource_analyze(ctx, url_or_file, connector):
|
|
|
280
214
|
for t in data:
|
|
281
215
|
click.echo(FeedbackManager.info_datasource_row(row=row_format.format(*[str(element) for element in t])))
|
|
282
216
|
|
|
283
|
-
analysis, _ = await _analyze(
|
|
284
|
-
url_or_file, client, format=get_format_from_filename_or_url(url_or_file), connector=_connector
|
|
285
|
-
)
|
|
217
|
+
analysis, _ = await _analyze(url_or_file, client, format=get_format_from_filename_or_url(url_or_file))
|
|
286
218
|
|
|
287
219
|
columns = ("name", "type", "nullable")
|
|
288
220
|
if "columns" in analysis["analysis"]:
|
|
@@ -513,30 +445,14 @@ async def datasource_delete_rows(ctx, datasource_name, sql_condition, yes, wait,
|
|
|
513
445
|
)
|
|
514
446
|
@click.argument("filenames", nargs=-1, default=None)
|
|
515
447
|
@click.option("--force", is_flag=True, default=False, help="Override existing files")
|
|
516
|
-
@click.option(
|
|
517
|
-
"--connector",
|
|
518
|
-
type=click.Choice(["bigquery", "snowflake"], case_sensitive=True),
|
|
519
|
-
help="Use from one of the selected connectors. In this case pass a table name as a parameter instead of a file name",
|
|
520
|
-
hidden=True,
|
|
521
|
-
)
|
|
522
448
|
@click.pass_context
|
|
523
449
|
@coro
|
|
524
|
-
async def generate_datasource(ctx: Context,
|
|
450
|
+
async def generate_datasource(ctx: Context, filenames, force: bool):
|
|
525
451
|
"""Generate a data source file based on a sample CSV file from local disk or url"""
|
|
526
452
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
527
453
|
|
|
528
|
-
_connector: Optional[Connector] = None
|
|
529
|
-
if connector:
|
|
530
|
-
load_connector_config(ctx, connector, False, check_uninstalled=False)
|
|
531
|
-
if connector not in ctx.ensure_object(dict):
|
|
532
|
-
raise CLIDatasourceException(FeedbackManager.error_connector_not_configured(connector=connector))
|
|
533
|
-
else:
|
|
534
|
-
_connector = ctx.ensure_object(dict)[connector]
|
|
535
|
-
|
|
536
454
|
for filename in filenames:
|
|
537
|
-
await _generate_datafile(
|
|
538
|
-
filename, client, force=force, format=get_format_from_filename_or_url(filename), connector=_connector
|
|
539
|
-
)
|
|
455
|
+
await _generate_datafile(filename, client, force=force, format=get_format_from_filename_or_url(filename))
|
|
540
456
|
|
|
541
457
|
|
|
542
458
|
@datasource.command(name="connect")
|
tinybird/tornado_template.py
CHANGED
|
@@ -214,6 +214,11 @@ from .context import disable_template_security_validation
|
|
|
214
214
|
_DEFAULT_AUTOESCAPE = "xhtml_escape"
|
|
215
215
|
_UNSET = object()
|
|
216
216
|
|
|
217
|
+
# Pre-compiled regex patterns for whitespace filtering (performance optimization)
|
|
218
|
+
_PATTERN_TAB_SPACE = re.compile(r"([\t ]+)")
|
|
219
|
+
_PATTERN_NEWLINE_SPACE = re.compile(r"(\s*\n\s*)")
|
|
220
|
+
_PATTERN_ALL_WHITESPACE = re.compile(r"(\s+)")
|
|
221
|
+
|
|
217
222
|
|
|
218
223
|
def filter_whitespace(mode, text):
|
|
219
224
|
"""Transform whitespace in ``text`` according to ``mode``.
|
|
@@ -231,11 +236,11 @@ def filter_whitespace(mode, text):
|
|
|
231
236
|
if mode == "all":
|
|
232
237
|
return text
|
|
233
238
|
elif mode == "single":
|
|
234
|
-
text =
|
|
235
|
-
text =
|
|
239
|
+
text = _PATTERN_TAB_SPACE.sub(" ", text)
|
|
240
|
+
text = _PATTERN_NEWLINE_SPACE.sub("\n", text)
|
|
236
241
|
return text
|
|
237
242
|
elif mode == "oneline":
|
|
238
|
-
return
|
|
243
|
+
return _PATTERN_ALL_WHITESPACE.sub(" ", text)
|
|
239
244
|
else:
|
|
240
245
|
raise Exception("invalid whitespace mode %s" % mode)
|
|
241
246
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird_cli
|
|
3
|
-
Version:
|
|
3
|
+
Version: 6.0.2.dev0
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/cli
|
|
6
6
|
Author: Tinybird
|
|
7
7
|
Author-email: support@tinybird.co
|
|
8
|
-
Requires-Python: >=3.
|
|
8
|
+
Requires-Python: >=3.10, <3.14
|
|
9
9
|
Description-Content-Type: text/x-rst
|
|
10
10
|
Requires-Dist: aiofiles==24.1.0
|
|
11
11
|
Requires-Dist: clickhouse-toolset==0.34.dev0
|
|
@@ -26,29 +26,11 @@ Requires-Dist: tornado~=6.0.0
|
|
|
26
26
|
Requires-Dist: urllib3<2,>=1.26.14
|
|
27
27
|
Requires-Dist: wheel
|
|
28
28
|
Requires-Dist: packaging<24,>=23.1
|
|
29
|
-
Provides-Extra: bigquery
|
|
30
|
-
Requires-Dist: gsutil==4.58; extra == "bigquery"
|
|
31
|
-
Requires-Dist: google-api-python-client==2.0.2; extra == "bigquery"
|
|
32
|
-
Requires-Dist: google-auth==1.27.1; extra == "bigquery"
|
|
33
|
-
Requires-Dist: google-auth-httplib2==0.1.0; extra == "bigquery"
|
|
34
|
-
Requires-Dist: google-cloud-storage==2.4.0; extra == "bigquery"
|
|
35
|
-
Requires-Dist: google-cloud-bigquery==2.11.0; extra == "bigquery"
|
|
36
|
-
Provides-Extra: snowflake
|
|
37
|
-
Requires-Dist: snowflake-connector-python~=3.12.3; extra == "snowflake"
|
|
38
|
-
Requires-Dist: gsutil==4.58; extra == "snowflake"
|
|
39
|
-
Requires-Dist: google-api-python-client==2.0.2; extra == "snowflake"
|
|
40
|
-
Requires-Dist: google-auth==1.27.1; extra == "snowflake"
|
|
41
|
-
Requires-Dist: google-auth-httplib2==0.1.0; extra == "snowflake"
|
|
42
|
-
Requires-Dist: google-cloud-storage==2.4.0; extra == "snowflake"
|
|
43
|
-
Requires-Dist: oauth2client==3.0.0; extra == "snowflake"
|
|
44
|
-
Requires-Dist: chardet<4,>=3.0.2; extra == "snowflake"
|
|
45
|
-
Requires-Dist: pyOpenSSL<20.0.0,>=16.2.0; extra == "snowflake"
|
|
46
29
|
Dynamic: author
|
|
47
30
|
Dynamic: author-email
|
|
48
31
|
Dynamic: description
|
|
49
32
|
Dynamic: description-content-type
|
|
50
33
|
Dynamic: home-page
|
|
51
|
-
Dynamic: provides-extra
|
|
52
34
|
Dynamic: requires-dist
|
|
53
35
|
Dynamic: requires-python
|
|
54
36
|
Dynamic: summary
|
|
@@ -61,6 +43,17 @@ The Tinybird command-line tool allows you to use all the Tinybird functionality
|
|
|
61
43
|
Changelog
|
|
62
44
|
----------
|
|
63
45
|
|
|
46
|
+
6.0.1
|
|
47
|
+
***********
|
|
48
|
+
|
|
49
|
+
- `Fixed` false circular dependency detection when a shared datasource has the same name as a local pipe
|
|
50
|
+
|
|
51
|
+
6.0.0
|
|
52
|
+
***********
|
|
53
|
+
|
|
54
|
+
- `Removed` support for Python 3.9. The minimum supported Python version is now 3.10.
|
|
55
|
+
- `Removed` BigQuery and Snowflake CDK connector commands from `tb connection create`
|
|
56
|
+
|
|
64
57
|
5.22.2
|
|
65
58
|
***********
|
|
66
59
|
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
tinybird/__cli__.py,sha256=QqnJj7GK-OKp_zB4arFD6USfuRL60-4dmLvRJtG1NOA,236
|
|
2
|
+
tinybird/check_pypi.py,sha256=_4NkharLyR_ELrAdit-ftqIWvOf7jZNPt3i76frlo9g,975
|
|
3
|
+
tinybird/client.py,sha256=I2ZKD0W2g4rhFNgWmoNlfjr50uRi1YAB0Q7ZZ0BQS3I,50527
|
|
4
|
+
tinybird/config.py,sha256=J-RdM_20QGfujU11e-Hq33Ius9E-3ecAzp7W8TWDXHA,7306
|
|
5
|
+
tinybird/context.py,sha256=o4yvlXPkMLmdh-XJl3wpmqPAMeRRz5ScKzKlHHKn_I8,1201
|
|
6
|
+
tinybird/datafile_common.py,sha256=jEgsipBzYHJfGAZGY6HUkyjDRPtxcupKhTLbN-ghLLo,232482
|
|
7
|
+
tinybird/datatypes.py,sha256=Ud_IphoDOMtTMzEsYIf2lO467EVWw_ctbsOnrEzDHvU,11359
|
|
8
|
+
tinybird/feedback_manager.py,sha256=OehfKVruCHwUNN1bHIbDICvOaIovc3hb6RjGHTyIkBc,67667
|
|
9
|
+
tinybird/git_settings.py,sha256=Sw_8rGmribEFJ4Z_6idrVytxpFYk7ez8ei0qHULzs3E,3934
|
|
10
|
+
tinybird/sql.py,sha256=gAjctWq8e6xQSlDpFL6LZZK9ZnsMtuzAOPP0wXcHlEI,48694
|
|
11
|
+
tinybird/sql_template.py,sha256=TiH8yWXeJPHIMTKn0jt5-rjsHfy9AuRlUvMyde_kdEc,128517
|
|
12
|
+
tinybird/sql_template_fmt.py,sha256=Ma4qcs-2r8ZXQC4GUmrCqYz34DsnGF8k5lE2Jwnr314,10638
|
|
13
|
+
tinybird/sql_toolset.py,sha256=Of1T8vwG6B8YKIdCS1OFgTEX-2BfVtC15NSd-K9EaSk,26190
|
|
14
|
+
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
15
|
+
tinybird/tb_cli.py,sha256=q1LGAsBVVMJsjR2HK62Pu6vpVtLzNmH8wHrEVUUdVkU,744
|
|
16
|
+
tinybird/tornado_template.py,sha256=czBv3rgnRenlK4os8riAiqO0Ynz-JXa3ACSr-FpxDmA,42095
|
|
17
|
+
tinybird/ch_utils/constants.py,sha256=yTNizMzgYNBzUc2EV3moBfdrDIggOe9hiuAgWF7sv2c,4333
|
|
18
|
+
tinybird/ch_utils/engine.py,sha256=xxBcnCvEyREpc5vfcvRLZV4jJyPyXYvbk1a-yf2TV0w,37517
|
|
19
|
+
tinybird/tb_cli_modules/auth.py,sha256=dRg9tOwIuPcumGOHn03U9fCT-sBvoCo14uj1_yK2T2w,8731
|
|
20
|
+
tinybird/tb_cli_modules/branch.py,sha256=92jKpb28yZMav4w5s6HboGty8ivtgSIFh1zxvLK3mBo,39348
|
|
21
|
+
tinybird/tb_cli_modules/cicd.py,sha256=0lMkb6CVOFZl5HOwgY8mK4T4mgI7O8335UngLXtCc-c,13851
|
|
22
|
+
tinybird/tb_cli_modules/cli.py,sha256=mcnqq0SkzeLHVHxrXgV3doc4bDzKsmGJcxqzG08Etzc,60217
|
|
23
|
+
tinybird/tb_cli_modules/common.py,sha256=XtiM6Cmxlj1X4fsAbi1CMx5MOiS09stK7xF5GcDkMqQ,77185
|
|
24
|
+
tinybird/tb_cli_modules/config.py,sha256=0kFDmsDcjKon32rgFGMHHKSbv4j5dOrXtVOlyuAyEkk,11510
|
|
25
|
+
tinybird/tb_cli_modules/connection.py,sha256=yoYUQo-Fl36LTHeGI3HpFOCLiP0wKhWsoP9P9G26NZ8,21305
|
|
26
|
+
tinybird/tb_cli_modules/datasource.py,sha256=DPUkK6rLAk9aRTBcWA6nzTi4AgDt2fJrm6TdynianwI,32528
|
|
27
|
+
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
28
|
+
tinybird/tb_cli_modules/fmt.py,sha256=edQap4tAqWMWogSIx5zriT75naLi73XTB3NwatmcrFw,3518
|
|
29
|
+
tinybird/tb_cli_modules/job.py,sha256=AG69LPb9MbobA1awwJFZJvxqarDKfRlsBjw2V1zvYqc,2964
|
|
30
|
+
tinybird/tb_cli_modules/pipe.py,sha256=LEPZ6EqQsW525W1UxuMMAGMMtNMJqxz5G3ZsCHO5XTc,31442
|
|
31
|
+
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
32
|
+
tinybird/tb_cli_modules/tag.py,sha256=9YHnruPnUNp1IJUe4qcSEMg9EbquIRo--Nxcsbvkvq8,3488
|
|
33
|
+
tinybird/tb_cli_modules/telemetry.py,sha256=W098H6jmS4kpE7hN3tadaREBTf7oMocel-lkKWN0pU8,10466
|
|
34
|
+
tinybird/tb_cli_modules/test.py,sha256=Vf8oK96V81HdKGsT79y6MUz6oz_VrYIwTbRnzzJs4rQ,4350
|
|
35
|
+
tinybird/tb_cli_modules/token.py,sha256=JXATKTlbXohP9ZDZjlz8E4VYG6zrknKZhuz_wh1zBBc,13793
|
|
36
|
+
tinybird/tb_cli_modules/workspace.py,sha256=yg3yb7_GniGJnOy2HqwEzePl47gQD-hywuQJyclZHi0,12455
|
|
37
|
+
tinybird/tb_cli_modules/workspace_members.py,sha256=ksXsjd233y9-sNlz4Qb-meZbX4zn1B84e_bSm2i8rhg,8731
|
|
38
|
+
tinybird/tb_cli_modules/tinyunit/tinyunit.py,sha256=4QlVoHE-O3akOBuIIJm3Mx2ewiZmPTT9vdgqIi9byTw,11267
|
|
39
|
+
tinybird/tb_cli_modules/tinyunit/tinyunit_lib.py,sha256=NHoXcCHPDcKWYLzgP3NViho3Ey-6RV-ynPDzySPrTPE,1817
|
|
40
|
+
tinybird_cli-6.0.2.dev0.dist-info/METADATA,sha256=uR0KOliyrhOe-2namsYqDVYezAHXpzN9HWgBRUFNeEY,80795
|
|
41
|
+
tinybird_cli-6.0.2.dev0.dist-info/WHEEL,sha256=beeZ86-EfXScwlR_HKu4SllMC9wUEj_8Z_4FJ3egI2w,91
|
|
42
|
+
tinybird_cli-6.0.2.dev0.dist-info/entry_points.txt,sha256=PKPKuPmA4IfJYnCFHHUiw-aAWZuBomFvwCklv1OyCjE,43
|
|
43
|
+
tinybird_cli-6.0.2.dev0.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
44
|
+
tinybird_cli-6.0.2.dev0.dist-info/RECORD,,
|