tinybird 0.0.1.dev196__py3-none-any.whl → 0.0.1.dev197__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/sql_toolset.py +39 -5
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/modules/common.py +6 -3
- tinybird/tb/modules/connection.py +230 -6
- tinybird/tb/modules/create.py +27 -7
- tinybird/tb/modules/datasource.py +55 -21
- {tinybird-0.0.1.dev196.dist-info → tinybird-0.0.1.dev197.dist-info}/METADATA +2 -1
- {tinybird-0.0.1.dev196.dist-info → tinybird-0.0.1.dev197.dist-info}/RECORD +11 -11
- {tinybird-0.0.1.dev196.dist-info → tinybird-0.0.1.dev197.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev196.dist-info → tinybird-0.0.1.dev197.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev196.dist-info → tinybird-0.0.1.dev197.dist-info}/top_level.txt +0 -0
tinybird/sql_toolset.py
CHANGED
|
@@ -227,25 +227,56 @@ def sql_get_used_tables(
|
|
|
227
227
|
|
|
228
228
|
|
|
229
229
|
class ReplacementsDict(dict):
|
|
230
|
+
def __init__(self, *args, enabled_table_functions=None, **kwargs):
|
|
231
|
+
self.enabled_table_functions = enabled_table_functions
|
|
232
|
+
super().__init__(*args, **kwargs)
|
|
233
|
+
|
|
230
234
|
def __getitem__(self, key):
|
|
231
235
|
v = super().__getitem__(key)
|
|
232
236
|
if isinstance(v, tuple):
|
|
233
237
|
k, r = v
|
|
234
238
|
if callable(r):
|
|
235
|
-
r = r()
|
|
239
|
+
r = update_callable_signature(r)(self.enabled_table_functions)
|
|
236
240
|
super().__setitem__(key, (k, r))
|
|
237
241
|
return k, r
|
|
238
242
|
if callable(v):
|
|
239
|
-
v = v()
|
|
243
|
+
v = update_callable_signature(v)(self.enabled_table_functions)
|
|
240
244
|
super().__setitem__(key, v)
|
|
241
245
|
return v
|
|
242
246
|
|
|
243
247
|
|
|
244
|
-
def
|
|
248
|
+
def update_callable_signature(func):
|
|
249
|
+
"""
|
|
250
|
+
Utility function to provide backward compatibility for callable functions
|
|
251
|
+
that don't accept the enabled_table_functions parameter.
|
|
252
|
+
"""
|
|
253
|
+
if callable(func):
|
|
254
|
+
|
|
255
|
+
def wrapper(enabled_table_functions=None):
|
|
256
|
+
# Check if the function accepts the enabled_table_functions parameter
|
|
257
|
+
import inspect
|
|
258
|
+
|
|
259
|
+
sig = inspect.signature(func)
|
|
260
|
+
if len(sig.parameters) == 0:
|
|
261
|
+
# Old-style callable with no parameters
|
|
262
|
+
return func()
|
|
263
|
+
else:
|
|
264
|
+
# New-style callable that accepts enabled_table_functions
|
|
265
|
+
return func(enabled_table_functions)
|
|
266
|
+
|
|
267
|
+
return wrapper
|
|
268
|
+
return func
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def tables_or_sql(replacement: dict, table_functions=False, function_allow_list=None) -> set:
|
|
245
272
|
try:
|
|
246
273
|
return set(
|
|
247
274
|
sql_get_used_tables(
|
|
248
|
-
replacement[1],
|
|
275
|
+
replacement[1],
|
|
276
|
+
default_database=replacement[0],
|
|
277
|
+
raising=True,
|
|
278
|
+
table_functions=table_functions,
|
|
279
|
+
function_allow_list=frozenset(function_allow_list),
|
|
249
280
|
)
|
|
250
281
|
)
|
|
251
282
|
except Exception as e:
|
|
@@ -342,6 +373,7 @@ def replace_tables(
|
|
|
342
373
|
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS
|
|
343
374
|
else:
|
|
344
375
|
_enabled_table_functions = ENABLED_TABLE_FUNCTIONS.union(set(function_allow_list))
|
|
376
|
+
_replacements.enabled_table_functions = frozenset(_enabled_table_functions)
|
|
345
377
|
while _tables:
|
|
346
378
|
table = _tables.pop()
|
|
347
379
|
if len(table) == 3:
|
|
@@ -355,7 +387,9 @@ def replace_tables(
|
|
|
355
387
|
seen_tables.add(table)
|
|
356
388
|
if table in _replacements:
|
|
357
389
|
replacement = _replacements[table]
|
|
358
|
-
dependent_tables = tables_or_sql(
|
|
390
|
+
dependent_tables = tables_or_sql(
|
|
391
|
+
replacement, table_functions=check_functions, function_allow_list=_enabled_table_functions
|
|
392
|
+
)
|
|
359
393
|
deps[table] |= {(d[0], d[1]) for d in dependent_tables}
|
|
360
394
|
for dependent_table in list(dependent_tables):
|
|
361
395
|
if len(dependent_table) == 3:
|
tinybird/tb/__cli__.py
CHANGED
|
@@ -4,5 +4,5 @@ __description__ = 'Tinybird Command Line Tool'
|
|
|
4
4
|
__url__ = 'https://www.tinybird.co/docs/forward/commands'
|
|
5
5
|
__author__ = 'Tinybird'
|
|
6
6
|
__author_email__ = 'support@tinybird.co'
|
|
7
|
-
__version__ = '0.0.1.
|
|
8
|
-
__revision__ = '
|
|
7
|
+
__version__ = '0.0.1.dev197'
|
|
8
|
+
__revision__ = '6bce0f0'
|
tinybird/tb/modules/common.py
CHANGED
|
@@ -1899,10 +1899,13 @@ def get_connection_name(project_folder: str, connection_type: str, connection_na
|
|
|
1899
1899
|
|
|
1900
1900
|
while not connection_name:
|
|
1901
1901
|
short_id = str(uuid.uuid4())[:4]
|
|
1902
|
+
default_name = f"{connection_type.lower()}_{short_id}"
|
|
1902
1903
|
connection_name = click.prompt(
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1904
|
+
FeedbackManager.highlight(
|
|
1905
|
+
message=f"? Connection name (only alphanumeric characters and underscores) [{default_name}]"
|
|
1906
|
+
),
|
|
1907
|
+
show_default=False,
|
|
1908
|
+
default=default_name,
|
|
1906
1909
|
)
|
|
1907
1910
|
assert isinstance(connection_name, str)
|
|
1908
1911
|
|
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
# - But please, **do not** interleave utility functions and command definitions.
|
|
5
5
|
|
|
6
6
|
import uuid
|
|
7
|
-
from typing import Any, Dict, List, Optional
|
|
7
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
8
8
|
|
|
9
9
|
import click
|
|
10
10
|
from click import Context
|
|
11
|
+
from confluent_kafka.admin import AdminClient
|
|
11
12
|
|
|
12
13
|
from tinybird.tb.client import TinyB
|
|
13
14
|
from tinybird.tb.modules.cli import cli
|
|
@@ -23,12 +24,18 @@ from tinybird.tb.modules.common import (
|
|
|
23
24
|
production_aws_iamrole_only,
|
|
24
25
|
run_aws_iamrole_connection_flow,
|
|
25
26
|
run_gcp_svc_account_connection_flow,
|
|
27
|
+
validate_kafka_auto_offset_reset,
|
|
28
|
+
validate_kafka_bootstrap_servers,
|
|
29
|
+
validate_kafka_key,
|
|
30
|
+
validate_kafka_schema_registry_url,
|
|
31
|
+
validate_kafka_secret,
|
|
26
32
|
)
|
|
27
33
|
from tinybird.tb.modules.create import (
|
|
28
34
|
generate_aws_iamrole_connection_file_with_secret,
|
|
29
35
|
generate_gcs_connection_file_with_secrets,
|
|
30
36
|
generate_kafka_connection_with_secrets,
|
|
31
37
|
)
|
|
38
|
+
from tinybird.tb.modules.exceptions import CLIConnectionException
|
|
32
39
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
33
40
|
from tinybird.tb.modules.project import Project
|
|
34
41
|
|
|
@@ -280,18 +287,235 @@ async def connection_create_gcs(ctx: Context) -> None:
|
|
|
280
287
|
|
|
281
288
|
|
|
282
289
|
@connection_create.command(name="kafka", short_help="Creates a Kafka connection.")
|
|
283
|
-
@click.option(
|
|
290
|
+
@click.option(
|
|
291
|
+
"--name",
|
|
292
|
+
default=None,
|
|
293
|
+
help="The name of your Kafka connection. If not provided, it's set as the bootstrap server",
|
|
294
|
+
)
|
|
295
|
+
@click.option("--bootstrap-servers", help="Kafka Bootstrap Server in form mykafka.mycloud.com:9092")
|
|
296
|
+
@click.option("--key", help="Key")
|
|
297
|
+
@click.option("--secret", help="Secret")
|
|
298
|
+
@click.option(
|
|
299
|
+
"--auto-offset-reset", default=None, help="Offset reset, can be 'latest' or 'earliest'. Defaults to 'latest'."
|
|
300
|
+
)
|
|
301
|
+
@click.option("--schema-registry-url", default=None, help="Avro Confluent Schema Registry URL")
|
|
302
|
+
@click.option(
|
|
303
|
+
"--sasl-mechanism",
|
|
304
|
+
default=None,
|
|
305
|
+
help="Authentication method for connection-based protocols. Defaults to 'PLAIN'",
|
|
306
|
+
)
|
|
307
|
+
@click.option(
|
|
308
|
+
"--security-protocol",
|
|
309
|
+
default="SASL_SSL",
|
|
310
|
+
help="Security protocol for connection-based protocols. Defaults to 'SASL_SSL'",
|
|
311
|
+
)
|
|
312
|
+
@click.option("--skip-secrets", is_flag=True, help="Skip Tinybird Secret creation")
|
|
284
313
|
@click.pass_context
|
|
285
314
|
@coro
|
|
286
|
-
async def
|
|
315
|
+
async def connection_create_kafka_cmd(
|
|
316
|
+
ctx: Context,
|
|
317
|
+
name: str,
|
|
318
|
+
bootstrap_servers: str,
|
|
319
|
+
key: str,
|
|
320
|
+
secret: str,
|
|
321
|
+
schema_registry_url: str,
|
|
322
|
+
auto_offset_reset: str,
|
|
323
|
+
sasl_mechanism: str,
|
|
324
|
+
security_protocol: str,
|
|
325
|
+
skip_secrets: bool,
|
|
326
|
+
) -> None:
|
|
287
327
|
"""
|
|
288
328
|
Creates a Kafka connection.
|
|
289
329
|
|
|
290
330
|
\b
|
|
291
331
|
$ tb connection create kafka
|
|
292
332
|
"""
|
|
293
|
-
|
|
333
|
+
await connection_create_kafka(
|
|
334
|
+
ctx,
|
|
335
|
+
name,
|
|
336
|
+
bootstrap_servers,
|
|
337
|
+
key,
|
|
338
|
+
secret,
|
|
339
|
+
schema_registry_url,
|
|
340
|
+
auto_offset_reset,
|
|
341
|
+
sasl_mechanism,
|
|
342
|
+
security_protocol,
|
|
343
|
+
skip_secrets,
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
async def connection_create_kafka(
|
|
348
|
+
ctx: Context,
|
|
349
|
+
name="",
|
|
350
|
+
bootstrap_servers="",
|
|
351
|
+
key="",
|
|
352
|
+
secret="",
|
|
353
|
+
schema_registry_url="",
|
|
354
|
+
auto_offset_reset="",
|
|
355
|
+
sasl_mechanism="",
|
|
356
|
+
security_protocol="SASL_SSL",
|
|
357
|
+
skip_secrets=False,
|
|
358
|
+
) -> Tuple[str, str, str, str, str, str, str, str, List[str]]:
|
|
359
|
+
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
360
|
+
obj: Dict[str, Any] = ctx.ensure_object(dict)
|
|
361
|
+
click.echo(FeedbackManager.gray(message="\n» Creating Kafka connection..."))
|
|
294
362
|
project: Project = ctx.ensure_object(dict)["project"]
|
|
295
363
|
name = get_kafka_connection_name(project.folder, name)
|
|
296
|
-
|
|
297
|
-
click.
|
|
364
|
+
default_bootstrap_servers = "localhost:9092"
|
|
365
|
+
bootstrap_servers = click.prompt(
|
|
366
|
+
FeedbackManager.highlight(
|
|
367
|
+
message=f"? Bootstrap servers (comma-separated list of host:port pairs) [{default_bootstrap_servers}]"
|
|
368
|
+
),
|
|
369
|
+
default=default_bootstrap_servers,
|
|
370
|
+
show_default=False,
|
|
371
|
+
)
|
|
372
|
+
if not bootstrap_servers:
|
|
373
|
+
bootstrap_servers = click.prompt("Bootstrap Server")
|
|
374
|
+
|
|
375
|
+
validate_kafka_bootstrap_servers(bootstrap_servers)
|
|
376
|
+
secret_required = not skip_secrets and click.confirm(
|
|
377
|
+
FeedbackManager.highlight(message=" ? Do you want to store the bootstrap server in a Tinybird Secret? [Y/n]"),
|
|
378
|
+
default=True,
|
|
379
|
+
show_default=False,
|
|
380
|
+
)
|
|
381
|
+
tb_secret_bootstrap_servers: Optional[str] = None
|
|
382
|
+
tb_secret_key: Optional[str] = None
|
|
383
|
+
tb_secret_secret: Optional[str] = None
|
|
384
|
+
|
|
385
|
+
if secret_required:
|
|
386
|
+
tb_secret_bootstrap_servers = str(click.prompt(FeedbackManager.highlight(message=" ? Secret name")))
|
|
387
|
+
try:
|
|
388
|
+
await client.create_secret(name=tb_secret_bootstrap_servers, value=bootstrap_servers)
|
|
389
|
+
except Exception as e:
|
|
390
|
+
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
391
|
+
|
|
392
|
+
if not key:
|
|
393
|
+
key = click.prompt(FeedbackManager.highlight(message="? Kafka key"))
|
|
394
|
+
|
|
395
|
+
validate_kafka_key(key)
|
|
396
|
+
|
|
397
|
+
secret_required = not skip_secrets and click.confirm(
|
|
398
|
+
FeedbackManager.highlight(message=" ? Do you want to store the Kafka key in a Tinybird Secret? [Y/n]"),
|
|
399
|
+
default=True,
|
|
400
|
+
show_default=False,
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
if secret_required:
|
|
404
|
+
tb_secret_key = str(click.prompt(FeedbackManager.highlight(message=" ? Secret name")))
|
|
405
|
+
try:
|
|
406
|
+
await client.create_secret(name=tb_secret_key, value=key)
|
|
407
|
+
except Exception as e:
|
|
408
|
+
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
409
|
+
|
|
410
|
+
if not secret:
|
|
411
|
+
secret = click.prompt(FeedbackManager.highlight(message="? Kafka secret"), hide_input=True)
|
|
412
|
+
|
|
413
|
+
validate_kafka_secret(secret)
|
|
414
|
+
|
|
415
|
+
secret_required = not skip_secrets and click.confirm(
|
|
416
|
+
FeedbackManager.highlight(message=" ? Do you want to store the Kafka secret in a Tinybird Secret? [Y/n]"),
|
|
417
|
+
default=True,
|
|
418
|
+
show_default=False,
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
if secret_required:
|
|
422
|
+
tb_secret_secret = str(click.prompt(FeedbackManager.highlight(message=" ? Secret name")))
|
|
423
|
+
try:
|
|
424
|
+
await client.create_secret(name=tb_secret_secret, value=secret)
|
|
425
|
+
except Exception as e:
|
|
426
|
+
raise CLIConnectionException(FeedbackManager.error(message=str(e)))
|
|
427
|
+
|
|
428
|
+
if not sasl_mechanism:
|
|
429
|
+
sasl_mechanism_options = ["PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512"]
|
|
430
|
+
sasl_mechanism = click.prompt(
|
|
431
|
+
FeedbackManager.highlight(message="? SASL Mechanism (PLAIN, SCRAM-SHA-256, SCRAM-SHA-512) [PLAIN]"),
|
|
432
|
+
type=click.Choice(sasl_mechanism_options),
|
|
433
|
+
show_default=False,
|
|
434
|
+
show_choices=False,
|
|
435
|
+
default="PLAIN",
|
|
436
|
+
)
|
|
437
|
+
|
|
438
|
+
create_in_cloud = (
|
|
439
|
+
click.confirm(
|
|
440
|
+
FeedbackManager.highlight(
|
|
441
|
+
message="? Would you like to create this connection in the cloud environment as well? [Y/n]"
|
|
442
|
+
),
|
|
443
|
+
default=True,
|
|
444
|
+
show_default=False,
|
|
445
|
+
)
|
|
446
|
+
if obj["env"] == "local"
|
|
447
|
+
and not skip_secrets
|
|
448
|
+
and (tb_secret_bootstrap_servers or tb_secret_key or tb_secret_secret)
|
|
449
|
+
else False
|
|
450
|
+
)
|
|
451
|
+
|
|
452
|
+
if create_in_cloud:
|
|
453
|
+
click.echo(FeedbackManager.gray(message="» Creating Secrets in cloud environment..."))
|
|
454
|
+
prod_config = obj["config"]
|
|
455
|
+
host = prod_config["host"]
|
|
456
|
+
token = prod_config["token"]
|
|
457
|
+
prod_client = TinyB(
|
|
458
|
+
token=token,
|
|
459
|
+
host=host,
|
|
460
|
+
staging=False,
|
|
461
|
+
)
|
|
462
|
+
if tb_secret_bootstrap_servers:
|
|
463
|
+
await prod_client.create_secret(name=tb_secret_bootstrap_servers, value=bootstrap_servers)
|
|
464
|
+
if tb_secret_key:
|
|
465
|
+
await prod_client.create_secret(name=tb_secret_key, value=key)
|
|
466
|
+
if tb_secret_secret:
|
|
467
|
+
await prod_client.create_secret(name=tb_secret_secret, value=secret)
|
|
468
|
+
click.echo(FeedbackManager.success(message="✓ Secrets created!"))
|
|
469
|
+
|
|
470
|
+
schema_registry_url and validate_kafka_schema_registry_url(schema_registry_url)
|
|
471
|
+
auto_offset_reset and validate_kafka_auto_offset_reset(auto_offset_reset)
|
|
472
|
+
click.echo(FeedbackManager.gray(message="» Validating connection..."))
|
|
473
|
+
topics = list_kafka_topics(bootstrap_servers, key, secret, security_protocol, sasl_mechanism)
|
|
474
|
+
|
|
475
|
+
if topics is None:
|
|
476
|
+
raise CLIConnectionException(FeedbackManager.error(message="Invalid Kafka connection"))
|
|
477
|
+
|
|
478
|
+
click.echo(FeedbackManager.success(message="✓ Connection is valid"))
|
|
479
|
+
await generate_kafka_connection_with_secrets(
|
|
480
|
+
name=name,
|
|
481
|
+
bootstrap_servers=bootstrap_servers,
|
|
482
|
+
tb_secret_bootstrap_servers=tb_secret_bootstrap_servers,
|
|
483
|
+
key=key,
|
|
484
|
+
tb_secret_key=tb_secret_key,
|
|
485
|
+
secret=secret,
|
|
486
|
+
tb_secret_secret=tb_secret_secret,
|
|
487
|
+
security_protocol=security_protocol,
|
|
488
|
+
sasl_mechanism=sasl_mechanism,
|
|
489
|
+
folder=project.folder,
|
|
490
|
+
)
|
|
491
|
+
click.echo(FeedbackManager.info(message=f"/connections/{name}.connection"))
|
|
492
|
+
click.echo(FeedbackManager.success(message="✓ Connection created!"))
|
|
493
|
+
return (
|
|
494
|
+
name,
|
|
495
|
+
bootstrap_servers,
|
|
496
|
+
key,
|
|
497
|
+
secret,
|
|
498
|
+
schema_registry_url,
|
|
499
|
+
auto_offset_reset,
|
|
500
|
+
sasl_mechanism,
|
|
501
|
+
security_protocol,
|
|
502
|
+
topics,
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def list_kafka_topics(bootstrap_servers, sasl_username, sasl_password, security_protocol, sasl_mechanism):
|
|
507
|
+
conf = {
|
|
508
|
+
"bootstrap.servers": bootstrap_servers,
|
|
509
|
+
"security.protocol": security_protocol,
|
|
510
|
+
"sasl.mechanism": sasl_mechanism,
|
|
511
|
+
"sasl.username": sasl_username,
|
|
512
|
+
"sasl.password": sasl_password,
|
|
513
|
+
"log_level": 0,
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
try:
|
|
517
|
+
client = AdminClient(conf)
|
|
518
|
+
metadata = client.list_topics(timeout=5)
|
|
519
|
+
return list(metadata.topics.keys())
|
|
520
|
+
except Exception:
|
|
521
|
+
return None
|
tinybird/tb/modules/create.py
CHANGED
|
@@ -540,12 +540,32 @@ TYPE ENDPOINT
|
|
|
540
540
|
return result
|
|
541
541
|
|
|
542
542
|
|
|
543
|
-
async def generate_kafka_connection_with_secrets(
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
543
|
+
async def generate_kafka_connection_with_secrets(
|
|
544
|
+
name: str,
|
|
545
|
+
bootstrap_servers: str,
|
|
546
|
+
key: str,
|
|
547
|
+
secret: str,
|
|
548
|
+
tb_secret_bootstrap_servers: Optional[str],
|
|
549
|
+
tb_secret_key: Optional[str],
|
|
550
|
+
tb_secret_secret: Optional[str],
|
|
551
|
+
security_protocol: str,
|
|
552
|
+
sasl_mechanism: str,
|
|
553
|
+
folder: str,
|
|
554
|
+
) -> Path:
|
|
555
|
+
kafka_bootstrap_servers = (
|
|
556
|
+
inject_tb_secret(tb_secret_bootstrap_servers) if tb_secret_bootstrap_servers else bootstrap_servers
|
|
557
|
+
)
|
|
558
|
+
kafka_key = inject_tb_secret(tb_secret_key) if tb_secret_key else key
|
|
559
|
+
kafka_secret = inject_tb_secret(tb_secret_secret) if tb_secret_secret else secret
|
|
560
|
+
content = f"""TYPE kafka
|
|
561
|
+
KAFKA_BOOTSTRAP_SERVERS {kafka_bootstrap_servers}
|
|
562
|
+
KAFKA_SECURITY_PROTOCOL {security_protocol or "SASL_SSL"}
|
|
563
|
+
KAFKA_SASL_MECHANISM {sasl_mechanism or "PLAIN"}
|
|
564
|
+
KAFKA_KEY {kafka_key}
|
|
565
|
+
KAFKA_SECRET {kafka_secret}
|
|
550
566
|
"""
|
|
551
567
|
return generate_connection_file(name, content, folder, skip_feedback=True)
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def inject_tb_secret(secret_name: str) -> str:
|
|
571
|
+
return f"""{{{{ tb_secret("{secret_name}") }}}}"""
|
|
@@ -10,7 +10,7 @@ import re
|
|
|
10
10
|
import uuid
|
|
11
11
|
from datetime import datetime
|
|
12
12
|
from pathlib import Path
|
|
13
|
-
from typing import Optional
|
|
13
|
+
from typing import List, Optional
|
|
14
14
|
from urllib.parse import urlparse
|
|
15
15
|
|
|
16
16
|
import click
|
|
@@ -33,11 +33,11 @@ from tinybird.tb.modules.common import (
|
|
|
33
33
|
push_data,
|
|
34
34
|
)
|
|
35
35
|
from tinybird.tb.modules.config import CLIConfig
|
|
36
|
+
from tinybird.tb.modules.connection import connection_create_kafka
|
|
36
37
|
from tinybird.tb.modules.create import (
|
|
37
38
|
create_resources_from_prompt,
|
|
38
39
|
generate_aws_iamrole_connection_file_with_secret,
|
|
39
40
|
generate_gcs_connection_file_with_secrets,
|
|
40
|
-
generate_kafka_connection_with_secrets,
|
|
41
41
|
)
|
|
42
42
|
from tinybird.tb.modules.datafile.common import get_name_version
|
|
43
43
|
from tinybird.tb.modules.datafile.fixture import persist_fixture
|
|
@@ -780,6 +780,8 @@ async def datasource_create(
|
|
|
780
780
|
return connection_files
|
|
781
781
|
|
|
782
782
|
connection_files = get_connection_files()
|
|
783
|
+
connection_name = ""
|
|
784
|
+
topics: List[str] = []
|
|
783
785
|
if len(connection_files) == 0:
|
|
784
786
|
click.echo(FeedbackManager.error(message=f"x No {datasource_type} connections found."))
|
|
785
787
|
if click.confirm(
|
|
@@ -787,15 +789,26 @@ async def datasource_create(
|
|
|
787
789
|
show_default=False,
|
|
788
790
|
default=True,
|
|
789
791
|
):
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
792
|
+
if datasource_type != "Kafka":
|
|
793
|
+
click.echo(FeedbackManager.gray(message="\n» Creating .connection file..."))
|
|
794
|
+
default_connection_name = f"{datasource_type.lower()}_{generate_short_id()}"
|
|
795
|
+
connection_name = click.prompt(
|
|
796
|
+
FeedbackManager.highlight(message=f"? Connection name [{default_connection_name}]"),
|
|
797
|
+
show_default=False,
|
|
798
|
+
default=default_connection_name,
|
|
799
|
+
)
|
|
797
800
|
if datasource_type == "Kafka":
|
|
798
|
-
|
|
801
|
+
(
|
|
802
|
+
connection_name,
|
|
803
|
+
bootstrap_servers,
|
|
804
|
+
key,
|
|
805
|
+
secret,
|
|
806
|
+
schema_registry_url,
|
|
807
|
+
auto_offset_reset,
|
|
808
|
+
sasl_mechanism,
|
|
809
|
+
security_protocol,
|
|
810
|
+
topics,
|
|
811
|
+
) = await connection_create_kafka(ctx)
|
|
799
812
|
elif datasource_type == "S3":
|
|
800
813
|
await generate_aws_iamrole_connection_file_with_secret(
|
|
801
814
|
connection_name,
|
|
@@ -812,8 +825,9 @@ async def datasource_create(
|
|
|
812
825
|
svc_account_creds="GCS_SERVICE_ACCOUNT_CREDENTIALS_JSON",
|
|
813
826
|
folder=project.folder,
|
|
814
827
|
)
|
|
815
|
-
|
|
816
|
-
|
|
828
|
+
if datasource_type != "Kafka":
|
|
829
|
+
click.echo(FeedbackManager.info(message=f"/connections/{connection_name}.connection"))
|
|
830
|
+
click.echo(FeedbackManager.success(message="✓ .connection created!"))
|
|
817
831
|
connection_files = get_connection_files()
|
|
818
832
|
else:
|
|
819
833
|
click.echo(
|
|
@@ -827,8 +841,8 @@ async def datasource_create(
|
|
|
827
841
|
connection = connection_path.stem
|
|
828
842
|
|
|
829
843
|
ds_content = """SCHEMA >
|
|
830
|
-
`timestamp` DateTime `json:$.timestamp`,
|
|
831
|
-
`session_id` String `json:$.session_id`
|
|
844
|
+
`timestamp` DateTime `json:$.timestamp`,
|
|
845
|
+
`session_id` String `json:$.session_id`
|
|
832
846
|
"""
|
|
833
847
|
|
|
834
848
|
if datasource_type == "Local file":
|
|
@@ -879,16 +893,36 @@ async def datasource_create(
|
|
|
879
893
|
if datasource_type == "Kafka":
|
|
880
894
|
connections = await client.connections("kafka")
|
|
881
895
|
connection_id = next((c["id"] for c in connections if c["name"] == connection), connection)
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
896
|
+
|
|
897
|
+
if not topics:
|
|
898
|
+
try:
|
|
899
|
+
topics = await client.kafka_list_topics(connection_id) if connection_id else []
|
|
900
|
+
except Exception:
|
|
901
|
+
topics = []
|
|
902
|
+
|
|
903
|
+
if len(topics) > 1:
|
|
904
|
+
click.echo(FeedbackManager.highlight(message="? Multiple topics found. Please select one:"))
|
|
905
|
+
topic_index = -1
|
|
906
|
+
while topic_index == -1:
|
|
907
|
+
for index, topic in enumerate(topics):
|
|
908
|
+
click.echo(f" [{index + 1}] {topic}")
|
|
909
|
+
topic_index = click.prompt("\nSelect option", default=1)
|
|
910
|
+
try:
|
|
911
|
+
topic = topics[int(topic_index) - 1]
|
|
912
|
+
except Exception:
|
|
913
|
+
topic_index = -1
|
|
914
|
+
else:
|
|
915
|
+
topic = topics[0] if len(topics) > 0 else "topic_0"
|
|
916
|
+
|
|
887
917
|
group_id = generate_kafka_group_id(topic)
|
|
918
|
+
ds_content = """SCHEMA >
|
|
919
|
+
`__value` String `json:$.__value`
|
|
920
|
+
"""
|
|
888
921
|
ds_content += f"""
|
|
889
922
|
KAFKA_CONNECTION_NAME {connection}
|
|
890
|
-
KAFKA_TOPIC {
|
|
891
|
-
KAFKA_GROUP_ID {
|
|
923
|
+
KAFKA_TOPIC {topic}
|
|
924
|
+
KAFKA_GROUP_ID {group_id}
|
|
925
|
+
KAFKA_STORE_RAW_VALUE 'True'
|
|
892
926
|
"""
|
|
893
927
|
|
|
894
928
|
if datasource_type == "S3":
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: tinybird
|
|
3
|
-
Version: 0.0.1.
|
|
3
|
+
Version: 0.0.1.dev197
|
|
4
4
|
Summary: Tinybird Command Line Tool
|
|
5
5
|
Home-page: https://www.tinybird.co/docs/forward/commands
|
|
6
6
|
Author: Tinybird
|
|
@@ -13,6 +13,7 @@ Requires-Dist: boto3
|
|
|
13
13
|
Requires-Dist: click<8.2,>=8.1.6
|
|
14
14
|
Requires-Dist: clickhouse-toolset==0.34.dev0
|
|
15
15
|
Requires-Dist: colorama==0.4.6
|
|
16
|
+
Requires-Dist: confluent-kafka==2.8.0
|
|
16
17
|
Requires-Dist: cryptography~=41.0.0
|
|
17
18
|
Requires-Dist: croniter==1.3.15
|
|
18
19
|
Requires-Dist: docker==7.1.0
|
|
@@ -7,12 +7,12 @@ tinybird/prompts.py,sha256=rpgvMpR103niDnoDMSz8hRAYBdgfrorfD5-7g23BMQQ,37596
|
|
|
7
7
|
tinybird/sql.py,sha256=C_B81wwv3BsqyXGhF5oTk9DcTUkrp7NwIFqSzd3Dmjc,47854
|
|
8
8
|
tinybird/sql_template.py,sha256=hWW8JawSWLl9GeWPYkC_Yrxj7P0MHEVMJ0Px9bedEgM,99817
|
|
9
9
|
tinybird/sql_template_fmt.py,sha256=KUHdj5rYCYm_rKKdXYSJAE9vIyXUQLB0YSZnUXHeBlY,10196
|
|
10
|
-
tinybird/sql_toolset.py,sha256=
|
|
10
|
+
tinybird/sql_toolset.py,sha256=M2rpLYkgV2W8NnYEYPC1tJdpy4uZHXVF64NBSKLQka4,19549
|
|
11
11
|
tinybird/syncasync.py,sha256=IPnOx6lMbf9SNddN1eBtssg8vCLHMt76SuZ6YNYm-Yk,27761
|
|
12
12
|
tinybird/tornado_template.py,sha256=jjNVDMnkYFWXflmT8KU_Ssbo5vR8KQq3EJMk5vYgXRw,41959
|
|
13
13
|
tinybird/ch_utils/constants.py,sha256=aYvg2C_WxYWsnqPdZB1ZFoIr8ZY-XjUXYyHKE9Ansj0,3890
|
|
14
14
|
tinybird/ch_utils/engine.py,sha256=X4tE9OrfaUy6kO9cqVEzyI9cDcmOF3IAssRRzsTsfEQ,40781
|
|
15
|
-
tinybird/tb/__cli__.py,sha256=
|
|
15
|
+
tinybird/tb/__cli__.py,sha256=tgXGyKWFAsuRn-lAx7DVyxXbZUHg8NZrCLGHWQZ3LS4,247
|
|
16
16
|
tinybird/tb/check_pypi.py,sha256=rW4QmDRbtgKdUUwJCnBkVjmTjZSZGN-XgZhx7vMkC0w,1009
|
|
17
17
|
tinybird/tb/cli.py,sha256=u3eGOhX0MHkuT6tiwaZ0_3twqLmqKXDAOxF7yV_Nn9Q,1075
|
|
18
18
|
tinybird/tb/client.py,sha256=CO-dQw8h28X6T6IO-Z79yPBKaJQT1Rwya5b6gexvw58,56491
|
|
@@ -20,12 +20,12 @@ tinybird/tb/config.py,sha256=jT9xndpeCY_g0HdB5qE2EquC0TFRRnkPnQFWZWd04jo,3998
|
|
|
20
20
|
tinybird/tb/modules/build.py,sha256=Deuq1Qk7c5Uapz92Vn9Bp7CrjDylRWER5hBvKGS3QxE,19370
|
|
21
21
|
tinybird/tb/modules/cicd.py,sha256=Njb6eZOHHbUkoJJx6KoixO9PsfA_T-3Ybkya9-50Ca8,7328
|
|
22
22
|
tinybird/tb/modules/cli.py,sha256=qmD1tNoPXxpDuTD_Vu4YIR9egc3m1eS0NIhaQzacOYo,15605
|
|
23
|
-
tinybird/tb/modules/common.py,sha256=
|
|
23
|
+
tinybird/tb/modules/common.py,sha256=R675BLZTnPlCoPphCTcy3S1lsf1MF_H3U_ox9yPNCqg,84187
|
|
24
24
|
tinybird/tb/modules/config.py,sha256=ziqW_t_mRVvWOd85VoB4vKyvgMkEfpXDf9H4v38p2xc,11422
|
|
25
|
-
tinybird/tb/modules/connection.py,sha256=
|
|
25
|
+
tinybird/tb/modules/connection.py,sha256=bULX6oru3MINhQGuXOmlALAJUTgLltO0kMOMtmLVNxg,17746
|
|
26
26
|
tinybird/tb/modules/copy.py,sha256=2Mm4FWKehOG7CoOhiF1m9UZJgJn0W1_cMolqju8ONYg,5805
|
|
27
|
-
tinybird/tb/modules/create.py,sha256=
|
|
28
|
-
tinybird/tb/modules/datasource.py,sha256=
|
|
27
|
+
tinybird/tb/modules/create.py,sha256=57oxoXl9a0swV7YFP43lcrmOChKaj9dcLcWrmiYDNyg,21543
|
|
28
|
+
tinybird/tb/modules/datasource.py,sha256=TUuX18Eno6tipnpqM16g8afesGTpTfFa4AK4lVmaLiU,38607
|
|
29
29
|
tinybird/tb/modules/deployment.py,sha256=QeLYvk6vNmOFC2FAvhR5tzFIYDkSCVgT08CGEIxr4fU,28154
|
|
30
30
|
tinybird/tb/modules/deprecations.py,sha256=rrszC1f_JJeJ8mUxGoCxckQTJFBCR8wREf4XXXN-PRc,4507
|
|
31
31
|
tinybird/tb/modules/dev_server.py,sha256=57FCKuWpErwYUYgHspYDkLWEm9F4pbvVOtMrFXX1fVU,10129
|
|
@@ -80,8 +80,8 @@ tinybird/tb_cli_modules/config.py,sha256=IsgdtFRnUrkY8-Zo32lmk6O7u3bHie1QCxLwgp4
|
|
|
80
80
|
tinybird/tb_cli_modules/exceptions.py,sha256=pmucP4kTF4irIt7dXiG-FcnI-o3mvDusPmch1L8RCWk,3367
|
|
81
81
|
tinybird/tb_cli_modules/regions.py,sha256=QjsL5H6Kg-qr0aYVLrvb1STeJ5Sx_sjvbOYO0LrEGMk,166
|
|
82
82
|
tinybird/tb_cli_modules/telemetry.py,sha256=Hh2Io8ZPROSunbOLuMvuIFU4TqwWPmQTqal4WS09K1A,10449
|
|
83
|
-
tinybird-0.0.1.
|
|
84
|
-
tinybird-0.0.1.
|
|
85
|
-
tinybird-0.0.1.
|
|
86
|
-
tinybird-0.0.1.
|
|
87
|
-
tinybird-0.0.1.
|
|
83
|
+
tinybird-0.0.1.dev197.dist-info/METADATA,sha256=x7RVK-yUWnX3SzbDhqYorhpvkepI1Zvlvc5trh0EqPg,1646
|
|
84
|
+
tinybird-0.0.1.dev197.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
85
|
+
tinybird-0.0.1.dev197.dist-info/entry_points.txt,sha256=LwdHU6TfKx4Qs7BqqtaczEZbImgU7Abe9Lp920zb_fo,43
|
|
86
|
+
tinybird-0.0.1.dev197.dist-info/top_level.txt,sha256=VqqqEmkAy7UNaD8-V51FCoMMWXjLUlR0IstvK7tJYVY,54
|
|
87
|
+
tinybird-0.0.1.dev197.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|