tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev235__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +6 -12
- tinybird/tb/modules/local_common.py +4 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev235.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev235.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/infra.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import json
|
|
3
2
|
import subprocess
|
|
4
3
|
import sys
|
|
@@ -13,7 +12,7 @@ from click import Context
|
|
|
13
12
|
|
|
14
13
|
from tinybird.tb.client import TinyB
|
|
15
14
|
from tinybird.tb.modules.cli import CLIException, cli
|
|
16
|
-
from tinybird.tb.modules.common import
|
|
15
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
|
|
17
16
|
from tinybird.tb.modules.config import CLIConfig
|
|
18
17
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
19
18
|
|
|
@@ -225,18 +224,18 @@ class Infrastructure:
|
|
|
225
224
|
def __init__(self, client: TinyB):
|
|
226
225
|
self.client = client
|
|
227
226
|
|
|
228
|
-
|
|
229
|
-
self.orgs =
|
|
227
|
+
def get_organizations_info(self, config: CLIConfig):
|
|
228
|
+
self.orgs = get_organizations_by_user(config)
|
|
230
229
|
|
|
231
|
-
|
|
230
|
+
def create_infra(self, name: str, host: str, organization_id: str) -> Dict[str, Any]:
|
|
232
231
|
"""Create a new infrastructure."""
|
|
233
|
-
infra =
|
|
232
|
+
infra = self.client.infra_create(organization_id=organization_id, name=name, host=host)
|
|
234
233
|
return infra
|
|
235
234
|
|
|
236
|
-
|
|
235
|
+
def list_infras(self) -> None:
|
|
237
236
|
"""List all self-managed regions for the admin organization."""
|
|
238
237
|
|
|
239
|
-
infras =
|
|
238
|
+
infras = self.get_infra_list()
|
|
240
239
|
columns = [
|
|
241
240
|
"name",
|
|
242
241
|
"host",
|
|
@@ -257,24 +256,24 @@ class Infrastructure:
|
|
|
257
256
|
echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
|
|
258
257
|
click.echo("\n")
|
|
259
258
|
|
|
260
|
-
|
|
259
|
+
def remove_infra(self, name: str):
|
|
261
260
|
try:
|
|
262
261
|
click.echo(FeedbackManager.highlight(message=f"» Deleting infrastructure '{name}' from Tinybird..."))
|
|
263
|
-
infras =
|
|
262
|
+
infras = self.get_infra_list()
|
|
264
263
|
infra = next((infra for infra in infras if infra["name"] == name), None)
|
|
265
264
|
if not infra:
|
|
266
265
|
raise CLIException(f"Infrastructure '{name}' not found")
|
|
267
|
-
|
|
266
|
+
self.client.infra_delete(infra["id"], infra["organization_id"])
|
|
268
267
|
click.echo(FeedbackManager.success(message=f"\n✓ Infrastructure '{name}' deleted"))
|
|
269
268
|
except Exception as e:
|
|
270
269
|
click.echo(FeedbackManager.error(message=f"✗ Error: {e}"))
|
|
271
270
|
|
|
272
|
-
|
|
273
|
-
infras_list =
|
|
271
|
+
def update(self, infra_name: str, name: str, host: str):
|
|
272
|
+
infras_list = self.get_infra_list()
|
|
274
273
|
infra = next((infra for infra in infras_list if infra["name"] == infra_name), None)
|
|
275
274
|
if not infra:
|
|
276
275
|
return None
|
|
277
|
-
|
|
276
|
+
self.client.infra_update(
|
|
278
277
|
infra_id=infra["id"],
|
|
279
278
|
organization_id=infra["organization_id"],
|
|
280
279
|
name=name,
|
|
@@ -282,7 +281,7 @@ class Infrastructure:
|
|
|
282
281
|
)
|
|
283
282
|
return infra
|
|
284
283
|
|
|
285
|
-
|
|
284
|
+
def get_infra_list(self) -> List[Dict[str, str]]:
|
|
286
285
|
"""Get a list of all infrastructures across organizations.
|
|
287
286
|
|
|
288
287
|
Returns:
|
|
@@ -298,7 +297,7 @@ class Infrastructure:
|
|
|
298
297
|
org_id = org.get("id") or ""
|
|
299
298
|
org_name = org.get("name")
|
|
300
299
|
try:
|
|
301
|
-
infras =
|
|
300
|
+
infras = self.client.infra_list(organization_id=org_id)
|
|
302
301
|
for infra in infras:
|
|
303
302
|
infra["organization"] = org_name
|
|
304
303
|
all_infras.append(infra)
|
|
@@ -322,7 +321,7 @@ def infra(ctx: Context) -> None:
|
|
|
322
321
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
323
322
|
config = CLIConfig.get_project_config()
|
|
324
323
|
infra = Infrastructure(client)
|
|
325
|
-
|
|
324
|
+
infra.get_organizations_info(config)
|
|
326
325
|
ctx.ensure_object(dict)["infra"] = infra
|
|
327
326
|
|
|
328
327
|
|
|
@@ -340,8 +339,7 @@ def infra(ctx: Context) -> None:
|
|
|
340
339
|
@click.option("--skip-apply", is_flag=True, help="Skip Terraform and kubectl configuration and application")
|
|
341
340
|
@click.option("--organization-id", type=str, help="Organization ID for the self-managed region")
|
|
342
341
|
@click.pass_context
|
|
343
|
-
|
|
344
|
-
async def infra_init(
|
|
342
|
+
def infra_init(
|
|
345
343
|
ctx: Context,
|
|
346
344
|
name: str,
|
|
347
345
|
cloud_provider: str,
|
|
@@ -380,7 +378,7 @@ async def infra_init(
|
|
|
380
378
|
config = {}
|
|
381
379
|
if config_path.exists():
|
|
382
380
|
try:
|
|
383
|
-
with open(config_path, "r") as f:
|
|
381
|
+
with open(config_path, "r") as f:
|
|
384
382
|
config = json.load(f)
|
|
385
383
|
click.echo(FeedbackManager.info(message="** Loaded existing configuration from config.json"))
|
|
386
384
|
except json.JSONDecodeError:
|
|
@@ -442,7 +440,7 @@ async def infra_init(
|
|
|
442
440
|
"kubernetes_context": kubernetes_context,
|
|
443
441
|
}
|
|
444
442
|
|
|
445
|
-
with open(config_path, "w") as f:
|
|
443
|
+
with open(config_path, "w") as f:
|
|
446
444
|
json.dump(config, f, indent=2)
|
|
447
445
|
|
|
448
446
|
click.echo(FeedbackManager.info(message=f"** Configuration saved to {config_path}"))
|
|
@@ -450,16 +448,16 @@ async def infra_init(
|
|
|
450
448
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
451
449
|
cli_config = CLIConfig.get_project_config()
|
|
452
450
|
|
|
453
|
-
infras =
|
|
451
|
+
infras = infra.get_infra_list()
|
|
454
452
|
infra_name = next((infra for infra in infras if infra["name"] == name), None)
|
|
455
453
|
if not infra_name:
|
|
456
454
|
# Handle organization if not provided
|
|
457
|
-
organization_id, organization_name =
|
|
455
|
+
organization_id, organization_name = ask_for_organization(infra.orgs, organization_id)
|
|
458
456
|
if not organization_id:
|
|
459
457
|
return
|
|
460
458
|
click.echo(FeedbackManager.highlight(message=f"\n» Creating infrastructure '{name}' in Tinybird..."))
|
|
461
459
|
host = f"https://{dns_record}.{dns_zone_name}"
|
|
462
|
-
infra_obj =
|
|
460
|
+
infra_obj = infra.create_infra(name, host, organization_id)
|
|
463
461
|
else:
|
|
464
462
|
click.echo(FeedbackManager.highlight(message=f"» Infrastructure '{name}' already exists."))
|
|
465
463
|
if infra_name["host"] != f"https://{dns_record}.{dns_zone_name}":
|
|
@@ -469,7 +467,7 @@ async def infra_init(
|
|
|
469
467
|
)
|
|
470
468
|
)
|
|
471
469
|
if click.confirm("Would you like to update the host in the infra provisioned at Tinybird?"):
|
|
472
|
-
|
|
470
|
+
infra.update(infra_name=name, name=name, host=f"https://{dns_record}.{dns_zone_name}")
|
|
473
471
|
organization_name = infra_name["organization"]
|
|
474
472
|
infra_obj = infra_name
|
|
475
473
|
|
|
@@ -480,7 +478,7 @@ async def infra_init(
|
|
|
480
478
|
"dns_record": dns_record,
|
|
481
479
|
}
|
|
482
480
|
|
|
483
|
-
with open(tf_path, "w") as f:
|
|
481
|
+
with open(tf_path, "w") as f:
|
|
484
482
|
f.write(terraform_content.lstrip())
|
|
485
483
|
|
|
486
484
|
click.echo(FeedbackManager.info(message=f"** Created Terraform configuration in {tf_path}"))
|
|
@@ -498,7 +496,7 @@ async def infra_init(
|
|
|
498
496
|
"infra_user": cli_config.get_user_email() or "",
|
|
499
497
|
}
|
|
500
498
|
|
|
501
|
-
with open(yaml_path, "w") as f:
|
|
499
|
+
with open(yaml_path, "w") as f:
|
|
502
500
|
f.write(new_content.lstrip())
|
|
503
501
|
|
|
504
502
|
click.echo(FeedbackManager.info(message=f"** Created Kubernetes configuration in {yaml_path}"))
|
|
@@ -510,7 +508,7 @@ async def infra_init(
|
|
|
510
508
|
command = f"terraform -chdir={infra_dir} init"
|
|
511
509
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
512
510
|
|
|
513
|
-
init_result = subprocess.run(["terraform", f"-chdir={infra_dir}", "init"], capture_output=True, text=True)
|
|
511
|
+
init_result = subprocess.run(["terraform", f"-chdir={infra_dir}", "init"], capture_output=True, text=True)
|
|
514
512
|
|
|
515
513
|
if init_result.returncode != 0:
|
|
516
514
|
click.echo(FeedbackManager.error(message="✗ Error: Terraform initialization failed:"))
|
|
@@ -521,7 +519,7 @@ async def infra_init(
|
|
|
521
519
|
click.echo(FeedbackManager.info(message="** Running Terraform plan..."))
|
|
522
520
|
command = f"terraform -chdir={infra_dir} plan"
|
|
523
521
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
524
|
-
plan_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
522
|
+
plan_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
525
523
|
|
|
526
524
|
if plan_result.returncode != 0:
|
|
527
525
|
click.echo(FeedbackManager.error(message="✗ Error: Terraform plan failed:"))
|
|
@@ -536,7 +534,7 @@ async def infra_init(
|
|
|
536
534
|
click.echo(FeedbackManager.info(message="** Applying Terraform configuration..."))
|
|
537
535
|
command = f"terraform -chdir={infra_dir} apply -auto-approve"
|
|
538
536
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
539
|
-
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
537
|
+
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
540
538
|
|
|
541
539
|
if apply_result.returncode != 0:
|
|
542
540
|
click.echo(FeedbackManager.error(message="✗ Error: Terraform apply failed:"))
|
|
@@ -553,7 +551,7 @@ async def infra_init(
|
|
|
553
551
|
# Get current kubectl context
|
|
554
552
|
command = "kubectl config current-context"
|
|
555
553
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
556
|
-
current_context_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
554
|
+
current_context_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
557
555
|
|
|
558
556
|
current_context = (
|
|
559
557
|
current_context_result.stdout.strip() if current_context_result.returncode == 0 else "unknown"
|
|
@@ -563,7 +561,7 @@ async def infra_init(
|
|
|
563
561
|
command = "kubectl config get-contexts -o name"
|
|
564
562
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
565
563
|
|
|
566
|
-
contexts_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
564
|
+
contexts_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
567
565
|
|
|
568
566
|
if contexts_result.returncode != 0:
|
|
569
567
|
click.echo(FeedbackManager.error(message="✗ Error: Failed to get kubectl contexts:"))
|
|
@@ -605,7 +603,7 @@ async def infra_init(
|
|
|
605
603
|
|
|
606
604
|
# Update the config with the selected context
|
|
607
605
|
config["kubernetes_context"] = selected_context
|
|
608
|
-
with open(config_path, "w") as f:
|
|
606
|
+
with open(config_path, "w") as f:
|
|
609
607
|
json.dump(config, f, indent=2)
|
|
610
608
|
click.echo(f"Updated configuration with selected Kubernetes context: {selected_context}")
|
|
611
609
|
|
|
@@ -616,7 +614,7 @@ async def infra_init(
|
|
|
616
614
|
command = f"kubectl --context {selected_context} diff -f {str(yaml_path)}"
|
|
617
615
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
618
616
|
|
|
619
|
-
diff_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
617
|
+
diff_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
620
618
|
|
|
621
619
|
if diff_result.returncode not in [0, 1]: # kubectl diff returns 1 when there are differences
|
|
622
620
|
if (
|
|
@@ -638,7 +636,7 @@ async def infra_init(
|
|
|
638
636
|
# Now apply the configuration
|
|
639
637
|
command = f"kubectl --context {selected_context} apply -f {str(yaml_path)}"
|
|
640
638
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
641
|
-
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
639
|
+
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
642
640
|
|
|
643
641
|
if apply_result.returncode != 0:
|
|
644
642
|
click.echo(FeedbackManager.error(message="✗ Error: Failed to apply Kubernetes configuration:"))
|
|
@@ -662,7 +660,7 @@ async def infra_init(
|
|
|
662
660
|
)
|
|
663
661
|
|
|
664
662
|
try:
|
|
665
|
-
response = requests.get(endpoint_url, allow_redirects=False, timeout=5)
|
|
663
|
+
response = requests.get(endpoint_url, allow_redirects=False, timeout=5)
|
|
666
664
|
if response.status_code < 400: # Consider any non-error response as success
|
|
667
665
|
click.echo(
|
|
668
666
|
"\n" + click.style("✅ HTTPS endpoint is now accessible!", fg="green", bold=True)
|
|
@@ -679,7 +677,7 @@ async def infra_init(
|
|
|
679
677
|
click.echo(" This might be due to DNS propagation or the Load Balancer provisioning delays")
|
|
680
678
|
click.echo(f" Try accessing {endpoint_url} manually in a few minutes")
|
|
681
679
|
else:
|
|
682
|
-
time.sleep(10)
|
|
680
|
+
time.sleep(10)
|
|
683
681
|
|
|
684
682
|
if not skip_apply:
|
|
685
683
|
# Print a summary with the endpoint URL
|
|
@@ -698,20 +696,18 @@ async def infra_init(
|
|
|
698
696
|
@infra.command(name="rm")
|
|
699
697
|
@click.argument("name")
|
|
700
698
|
@click.pass_context
|
|
701
|
-
|
|
702
|
-
async def infra_remove(ctx: click.Context, name: str):
|
|
699
|
+
def infra_remove(ctx: click.Context, name: str):
|
|
703
700
|
"""Delete an infrastructure from Tinybird"""
|
|
704
701
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
705
|
-
|
|
702
|
+
infra.remove_infra(name)
|
|
706
703
|
|
|
707
704
|
|
|
708
705
|
@infra.command(name="ls")
|
|
709
706
|
@click.pass_context
|
|
710
|
-
|
|
711
|
-
async def infra_list(ctx: click.Context):
|
|
707
|
+
def infra_list(ctx: click.Context):
|
|
712
708
|
"""List self-managed infrastructures"""
|
|
713
709
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
714
|
-
|
|
710
|
+
infra.list_infras()
|
|
715
711
|
|
|
716
712
|
|
|
717
713
|
@infra.command(name="add")
|
|
@@ -723,7 +719,7 @@ def infra_add(ctx: click.Context, name: str, host: Optional[str] = None, organiz
|
|
|
723
719
|
"""Creates a new self-managed region from an existing infrastructure URL."""
|
|
724
720
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
725
721
|
|
|
726
|
-
organization_id, organization_name =
|
|
722
|
+
organization_id, organization_name = ask_for_organization(infra.orgs, organization_id)
|
|
727
723
|
if not organization_id:
|
|
728
724
|
return
|
|
729
725
|
|
|
@@ -736,7 +732,7 @@ def infra_add(ctx: click.Context, name: str, host: Optional[str] = None, organiz
|
|
|
736
732
|
)
|
|
737
733
|
|
|
738
734
|
click.echo(FeedbackManager.highlight(message=f"» Adding self-managed region '{name}' in Tinybird..."))
|
|
739
|
-
new_infra =
|
|
735
|
+
new_infra = infra.create_infra(name, host, organization_id)
|
|
740
736
|
infra_token = new_infra["token"]
|
|
741
737
|
click.echo(
|
|
742
738
|
FeedbackManager.success(message=f"\n✓ Self-managed region '{name}' added in '{organization_name}' Organization")
|
|
@@ -758,8 +754,7 @@ def infra_add(ctx: click.Context, name: str, host: Optional[str] = None, organiz
|
|
|
758
754
|
@click.option("--name", type=str, help="Name for identifying the self-managed region in Tinybird")
|
|
759
755
|
@click.option("--host", type=str, help="Host for the self-managed region")
|
|
760
756
|
@click.pass_context
|
|
761
|
-
|
|
762
|
-
async def infra_update(ctx: click.Context, infra_name: str, name: str, host: str):
|
|
757
|
+
def infra_update(ctx: click.Context, infra_name: str, name: str, host: str):
|
|
763
758
|
"""Updates the URL of an existing self-managed region."""
|
|
764
759
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
765
760
|
if not name and not host:
|
|
@@ -771,7 +766,7 @@ async def infra_update(ctx: click.Context, infra_name: str, name: str, host: str
|
|
|
771
766
|
click.echo(
|
|
772
767
|
FeedbackManager.highlight(message=f"» Updating self-managed region'{infra_name}' in Tinybird...")
|
|
773
768
|
)
|
|
774
|
-
infra_id =
|
|
769
|
+
infra_id = infra.update(infra_name, name, host)
|
|
775
770
|
if not infra_id:
|
|
776
771
|
raise CLIException(f"Self-managed region '{infra_name}' not found")
|
|
777
772
|
except Exception as e:
|
tinybird/tb/modules/job.py
CHANGED
|
@@ -10,7 +10,7 @@ from click import Context
|
|
|
10
10
|
|
|
11
11
|
from tinybird.tb.client import DoesNotExistException, TinyB
|
|
12
12
|
from tinybird.tb.modules.cli import cli
|
|
13
|
-
from tinybird.tb.modules.common import
|
|
13
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
|
|
14
14
|
from tinybird.tb.modules.exceptions import CLIException
|
|
15
15
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
16
16
|
|
|
@@ -38,11 +38,10 @@ def job(ctx: Context) -> None:
|
|
|
38
38
|
default=None,
|
|
39
39
|
)
|
|
40
40
|
@click.pass_context
|
|
41
|
-
|
|
42
|
-
async def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...]) -> None:
|
|
41
|
+
def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...]) -> None:
|
|
43
42
|
"""List jobs, up to 100"""
|
|
44
43
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
45
|
-
jobs =
|
|
44
|
+
jobs = client.jobs(status=status, kind=kind)
|
|
46
45
|
columns = ["id", "kind", "status", "created at", "updated at", "job url"]
|
|
47
46
|
click.echo(FeedbackManager.info_jobs())
|
|
48
47
|
table = []
|
|
@@ -55,11 +54,10 @@ async def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...])
|
|
|
55
54
|
@job.command(name="details")
|
|
56
55
|
@click.argument("job_id")
|
|
57
56
|
@click.pass_context
|
|
58
|
-
|
|
59
|
-
async def job_details(ctx: Context, job_id: str) -> None:
|
|
57
|
+
def job_details(ctx: Context, job_id: str) -> None:
|
|
60
58
|
"""Get details for any job created in the last 48h"""
|
|
61
59
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
62
|
-
job =
|
|
60
|
+
job = client.job(job_id)
|
|
63
61
|
columns = []
|
|
64
62
|
click.echo(FeedbackManager.info_job(job=job_id))
|
|
65
63
|
table = []
|
|
@@ -72,13 +70,12 @@ async def job_details(ctx: Context, job_id: str) -> None:
|
|
|
72
70
|
@job.command(name="cancel")
|
|
73
71
|
@click.argument("job_id")
|
|
74
72
|
@click.pass_context
|
|
75
|
-
|
|
76
|
-
async def job_cancel(ctx: Context, job_id: str) -> None:
|
|
73
|
+
def job_cancel(ctx: Context, job_id: str) -> None:
|
|
77
74
|
"""Try to cancel a job"""
|
|
78
75
|
client = ctx.ensure_object(dict)["client"]
|
|
79
76
|
|
|
80
77
|
try:
|
|
81
|
-
result =
|
|
78
|
+
result = client.job_cancel(job_id)
|
|
82
79
|
except DoesNotExistException:
|
|
83
80
|
raise CLIException(FeedbackManager.error_job_does_not_exist(job_id=job_id))
|
|
84
81
|
except Exception as e:
|
tinybird/tb/modules/local.py
CHANGED
|
@@ -5,7 +5,6 @@ import requests
|
|
|
5
5
|
|
|
6
6
|
from docker.client import DockerClient
|
|
7
7
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.common import coro
|
|
9
8
|
from tinybird.tb.modules.exceptions import CLIException
|
|
10
9
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
11
10
|
from tinybird.tb.modules.local_common import (
|
|
@@ -88,8 +87,7 @@ def local(ctx: click.Context) -> None:
|
|
|
88
87
|
|
|
89
88
|
|
|
90
89
|
@local.command()
|
|
91
|
-
|
|
92
|
-
async def stop() -> None:
|
|
90
|
+
def stop() -> None:
|
|
93
91
|
"""Stop Tinybird Local"""
|
|
94
92
|
click.echo(FeedbackManager.highlight(message="» Shutting down Tinybird Local..."))
|
|
95
93
|
docker_client = get_docker_client()
|
|
@@ -99,8 +97,7 @@ async def stop() -> None:
|
|
|
99
97
|
|
|
100
98
|
@local.command()
|
|
101
99
|
@click.pass_context
|
|
102
|
-
|
|
103
|
-
async def status(ctx: click.Context) -> None:
|
|
100
|
+
def status(ctx: click.Context) -> None:
|
|
104
101
|
"""Check status of Tinybird Local"""
|
|
105
102
|
docker_client = get_docker_client()
|
|
106
103
|
container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, {})
|
|
@@ -115,7 +112,7 @@ async def status(ctx: click.Context) -> None:
|
|
|
115
112
|
from tinybird.tb.modules.info import get_local_info
|
|
116
113
|
|
|
117
114
|
config = ctx.ensure_object(dict).get("config", {})
|
|
118
|
-
|
|
115
|
+
get_local_info(config)
|
|
119
116
|
elif status == "restarting" or (status == "running" and health == "starting"):
|
|
120
117
|
click.echo(FeedbackManager.highlight(message="* Tinybird Local is starting..."))
|
|
121
118
|
elif status == "removing":
|
|
@@ -129,8 +126,7 @@ async def status(ctx: click.Context) -> None:
|
|
|
129
126
|
|
|
130
127
|
|
|
131
128
|
@local.command()
|
|
132
|
-
|
|
133
|
-
async def remove() -> None:
|
|
129
|
+
def remove() -> None:
|
|
134
130
|
"""Remove Tinybird Local"""
|
|
135
131
|
click.echo(FeedbackManager.highlight(message="» Removing Tinybird Local..."))
|
|
136
132
|
docker_client = get_docker_client()
|
|
@@ -139,14 +135,13 @@ async def remove() -> None:
|
|
|
139
135
|
|
|
140
136
|
|
|
141
137
|
@local.command()
|
|
142
|
-
@coro
|
|
143
138
|
@click.option(
|
|
144
139
|
"--use-aws-creds",
|
|
145
140
|
default=False,
|
|
146
141
|
is_flag=True,
|
|
147
142
|
help="Use local AWS credentials from your environment and pass them to the Tinybird docker container",
|
|
148
143
|
)
|
|
149
|
-
|
|
144
|
+
def start(use_aws_creds: bool) -> None:
|
|
150
145
|
"""Start Tinybird Local"""
|
|
151
146
|
click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
|
|
152
147
|
docker_client = get_docker_client()
|
|
@@ -155,14 +150,13 @@ async def start(use_aws_creds: bool) -> None:
|
|
|
155
150
|
|
|
156
151
|
|
|
157
152
|
@local.command()
|
|
158
|
-
@coro
|
|
159
153
|
@click.option(
|
|
160
154
|
"--use-aws-creds",
|
|
161
155
|
default=False,
|
|
162
156
|
is_flag=True,
|
|
163
157
|
help="Use local AWS credentials from your environment and pass them to the Tinybird docker container",
|
|
164
158
|
)
|
|
165
|
-
|
|
159
|
+
def restart(use_aws_creds: bool) -> None:
|
|
166
160
|
"""Restart Tinybird Local"""
|
|
167
161
|
click.echo(FeedbackManager.highlight(message="» Restarting Tinybird Local..."))
|
|
168
162
|
docker_client = get_docker_client()
|
|
@@ -28,16 +28,16 @@ TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
|
|
|
28
28
|
TB_LOCAL_DEFAULT_WORKSPACE_NAME = "Tinybird_Local_Testing"
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
def get_tinybird_local_client(
|
|
32
32
|
config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
|
|
33
33
|
) -> TinyB:
|
|
34
34
|
"""Get a Tinybird client connected to the local environment."""
|
|
35
35
|
|
|
36
|
-
config =
|
|
36
|
+
config = get_tinybird_local_config(config_obj, test=test, silent=silent)
|
|
37
37
|
return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
|
|
41
41
|
"""Craft a client config with a workspace name based on the path of the project files
|
|
42
42
|
|
|
43
43
|
It uses the tokens from tinybird local
|
|
@@ -76,7 +76,7 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = Fal
|
|
|
76
76
|
ws = None
|
|
77
77
|
|
|
78
78
|
if not ws:
|
|
79
|
-
|
|
79
|
+
user_client.create_workspace(ws_name, assign_to_organization_id=user_org_id, version="v1")
|
|
80
80
|
user_workspaces = requests.get(f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?token={admin_token}").json()
|
|
81
81
|
ws = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == ws_name), None)
|
|
82
82
|
if not ws:
|
tinybird/tb/modules/login.py
CHANGED
|
@@ -19,7 +19,7 @@ import requests
|
|
|
19
19
|
|
|
20
20
|
from tinybird.tb.config import DEFAULT_API_HOST
|
|
21
21
|
from tinybird.tb.modules.cli import CLIConfig, cli
|
|
22
|
-
from tinybird.tb.modules.common import ask_for_region_interactively,
|
|
22
|
+
from tinybird.tb.modules.common import ask_for_region_interactively, get_regions
|
|
23
23
|
from tinybird.tb.modules.exceptions import CLILoginException
|
|
24
24
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
25
25
|
|
|
@@ -135,8 +135,7 @@ def start_server(auth_callback, auth_host):
|
|
|
135
135
|
default="browser",
|
|
136
136
|
help="Set the authentication method to use. Default: browser.",
|
|
137
137
|
)
|
|
138
|
-
|
|
139
|
-
async def login(host: Optional[str], auth_host: str, workspace: str, interactive: bool, method: str):
|
|
138
|
+
def login(host: Optional[str], auth_host: str, workspace: str, interactive: bool, method: str):
|
|
140
139
|
"""Authenticate using the browser."""
|
|
141
140
|
try:
|
|
142
141
|
cli_config = CLIConfig.get_project_config()
|
|
@@ -148,7 +147,7 @@ async def login(host: Optional[str], auth_host: str, workspace: str, interactive
|
|
|
148
147
|
else:
|
|
149
148
|
click.echo(FeedbackManager.highlight(message="» No region detected, select one from the list below:"))
|
|
150
149
|
|
|
151
|
-
regions =
|
|
150
|
+
regions = get_regions(cli_config)
|
|
152
151
|
selected_region = ask_for_region_interactively(regions)
|
|
153
152
|
|
|
154
153
|
# If the user cancels the selection, we'll exit
|
|
@@ -202,7 +201,7 @@ async def login(host: Optional[str], auth_host: str, workspace: str, interactive
|
|
|
202
201
|
|
|
203
202
|
time.sleep(2) # noqa: ASYNC251
|
|
204
203
|
|
|
205
|
-
|
|
204
|
+
poll_for_tokens()
|
|
206
205
|
return
|
|
207
206
|
|
|
208
207
|
auth_event = threading.Event()
|
|
@@ -236,12 +235,12 @@ async def login(host: Optional[str], auth_host: str, workspace: str, interactive
|
|
|
236
235
|
if auth_event.wait(timeout=SERVER_MAX_WAIT_TIME): # Wait for up to 180 seconds
|
|
237
236
|
params = {}
|
|
238
237
|
params["code"] = auth_code[0]
|
|
239
|
-
response = requests.get(
|
|
238
|
+
response = requests.get(
|
|
240
239
|
f"{auth_host}/api/cli-login?{urlencode(params)}",
|
|
241
240
|
)
|
|
242
241
|
|
|
243
242
|
data = response.json()
|
|
244
|
-
|
|
243
|
+
authenticate_with_tokens(data, host, cli_config)
|
|
245
244
|
else:
|
|
246
245
|
raise Exception("Authentication failed or timed out.")
|
|
247
246
|
except Exception as e:
|
|
@@ -308,13 +307,13 @@ def create_one_time_code():
|
|
|
308
307
|
return seperator.join(parts), full_code
|
|
309
308
|
|
|
310
309
|
|
|
311
|
-
|
|
310
|
+
def authenticate_with_tokens(data: Dict[str, Any], host: Optional[str], cli_config: CLIConfig):
|
|
312
311
|
cli_config.set_token(data.get("workspace_token", ""))
|
|
313
312
|
host = host or data.get("api_host", "")
|
|
314
313
|
cli_config.set_token_for_host(data.get("workspace_token", ""), host)
|
|
315
314
|
cli_config.set_user_token(data.get("user_token", ""))
|
|
316
315
|
cli_config.set_host(host)
|
|
317
|
-
ws =
|
|
316
|
+
ws = cli_config.get_client(token=data.get("workspace_token", ""), host=host).workspace_info(version="v1")
|
|
318
317
|
for k in ("id", "name", "user_email", "user_id", "scope"):
|
|
319
318
|
if k in ws:
|
|
320
319
|
cli_config[k] = ws[k]
|
|
@@ -322,7 +321,7 @@ async def authenticate_with_tokens(data: Dict[str, Any], host: Optional[str], cl
|
|
|
322
321
|
path = os.path.join(os.getcwd(), ".tinyb")
|
|
323
322
|
cli_config.persist_to_file(override_with_path=path)
|
|
324
323
|
|
|
325
|
-
auth_info: Dict[str, Any] =
|
|
324
|
+
auth_info: Dict[str, Any] = cli_config.get_user_client().check_auth_login()
|
|
326
325
|
if not auth_info.get("is_valid", False):
|
|
327
326
|
raise Exception(FeedbackManager.error_auth_login_not_valid(host=cli_config.get_host()))
|
|
328
327
|
|
|
@@ -7,7 +7,6 @@ from tinybird.datafile.common import PipeTypes, get_name_version
|
|
|
7
7
|
from tinybird.tb.client import TinyB
|
|
8
8
|
from tinybird.tb.modules.cli import cli
|
|
9
9
|
from tinybird.tb.modules.common import (
|
|
10
|
-
coro,
|
|
11
10
|
echo_safe_humanfriendly_tables_format_smart_table,
|
|
12
11
|
wait_job,
|
|
13
12
|
)
|
|
@@ -31,15 +30,14 @@ def materialization(ctx):
|
|
|
31
30
|
help="Force a type of the output",
|
|
32
31
|
)
|
|
33
32
|
@click.pass_context
|
|
34
|
-
|
|
35
|
-
async def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
33
|
+
def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
36
34
|
"""List materializations"""
|
|
37
35
|
|
|
38
36
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
39
|
-
pipes =
|
|
37
|
+
pipes = client.pipes(dependencies=True, node_attrs="name,materialized", attrs="name,updated_at,endpoint,type")
|
|
40
38
|
materializations = [p for p in pipes if p.get("type") == PipeTypes.MATERIALIZED]
|
|
41
39
|
materializations = sorted(materializations, key=lambda p: p["updated_at"])
|
|
42
|
-
datasources =
|
|
40
|
+
datasources = client.datasources()
|
|
43
41
|
columns = ["name", "updated at", "nodes", "target datasource"]
|
|
44
42
|
table_human_readable = []
|
|
45
43
|
table_machine_readable = []
|
|
@@ -90,8 +88,7 @@ async def materialization_ls(ctx: click.Context, match: str, format_: str):
|
|
|
90
88
|
help="Waits for populate jobs to finish, showing a progress bar. Disabled by default.",
|
|
91
89
|
)
|
|
92
90
|
@click.pass_context
|
|
93
|
-
|
|
94
|
-
async def pipe_populate(
|
|
91
|
+
def pipe_populate(
|
|
95
92
|
ctx: click.Context,
|
|
96
93
|
pipe_name: str,
|
|
97
94
|
node: str,
|
|
@@ -108,7 +105,7 @@ async def pipe_populate(
|
|
|
108
105
|
|
|
109
106
|
cl: TinyB = ctx.ensure_object(dict)["client"]
|
|
110
107
|
|
|
111
|
-
pipe =
|
|
108
|
+
pipe = cl.pipe(pipe_name)
|
|
112
109
|
|
|
113
110
|
if pipe["type"] != PipeTypes.MATERIALIZED:
|
|
114
111
|
raise CLIPipeException(FeedbackManager.error_pipe_not_materialized(pipe=pipe_name))
|
|
@@ -124,7 +121,7 @@ async def pipe_populate(
|
|
|
124
121
|
|
|
125
122
|
node = materialized_ids[0]
|
|
126
123
|
|
|
127
|
-
response =
|
|
124
|
+
response = cl.populate_node(pipe_name, node, populate_condition=sql_condition, truncate=truncate)
|
|
128
125
|
if "job" not in response:
|
|
129
126
|
raise CLIPipeException(response)
|
|
130
127
|
|
|
@@ -135,4 +132,4 @@ async def pipe_populate(
|
|
|
135
132
|
else:
|
|
136
133
|
click.echo(FeedbackManager.info_populate_job_url(url=job_url))
|
|
137
134
|
if wait:
|
|
138
|
-
|
|
135
|
+
wait_job(cl, job_id, job_url, "Populating")
|