tinybird 0.0.1.dev234__py3-none-any.whl → 0.0.1.dev236__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of tinybird might be problematic. Click here for more details.
- tinybird/tb/__cli__.py +2 -2
- tinybird/tb/check_pypi.py +3 -8
- tinybird/tb/cli.py +0 -6
- tinybird/tb/client.py +314 -340
- tinybird/tb/config.py +4 -5
- tinybird/tb/modules/build.py +21 -24
- tinybird/tb/modules/cicd.py +2 -2
- tinybird/tb/modules/cli.py +18 -28
- tinybird/tb/modules/common.py +123 -138
- tinybird/tb/modules/config.py +2 -4
- tinybird/tb/modules/connection.py +21 -26
- tinybird/tb/modules/copy.py +7 -9
- tinybird/tb/modules/create.py +18 -21
- tinybird/tb/modules/datafile/build.py +39 -39
- tinybird/tb/modules/datafile/build_common.py +9 -9
- tinybird/tb/modules/datafile/build_datasource.py +24 -24
- tinybird/tb/modules/datafile/build_pipe.py +11 -13
- tinybird/tb/modules/datafile/diff.py +12 -12
- tinybird/tb/modules/datafile/format_datasource.py +5 -5
- tinybird/tb/modules/datafile/format_pipe.py +6 -6
- tinybird/tb/modules/datafile/playground.py +42 -42
- tinybird/tb/modules/datafile/pull.py +24 -26
- tinybird/tb/modules/datasource.py +42 -56
- tinybird/tb/modules/endpoint.py +14 -19
- tinybird/tb/modules/info.py +14 -15
- tinybird/tb/modules/infra.py +43 -48
- tinybird/tb/modules/job.py +7 -10
- tinybird/tb/modules/local.py +22 -18
- tinybird/tb/modules/local_common.py +13 -4
- tinybird/tb/modules/login.py +9 -10
- tinybird/tb/modules/materialization.py +7 -10
- tinybird/tb/modules/mock.py +8 -9
- tinybird/tb/modules/open.py +1 -3
- tinybird/tb/modules/pipe.py +2 -4
- tinybird/tb/modules/secret.py +12 -16
- tinybird/tb/modules/shell.py +7 -20
- tinybird/tb/modules/sink.py +6 -8
- tinybird/tb/modules/test.py +9 -14
- tinybird/tb/modules/tinyunit/tinyunit.py +3 -3
- tinybird/tb/modules/token.py +16 -24
- tinybird/tb/modules/watch.py +3 -7
- tinybird/tb/modules/workspace.py +26 -37
- tinybird/tb/modules/workspace_members.py +16 -23
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/METADATA +1 -1
- tinybird-0.0.1.dev236.dist-info/RECORD +89 -0
- tinybird-0.0.1.dev234.dist-info/RECORD +0 -89
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/WHEEL +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/entry_points.txt +0 -0
- {tinybird-0.0.1.dev234.dist-info → tinybird-0.0.1.dev236.dist-info}/top_level.txt +0 -0
tinybird/tb/modules/infra.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import json
|
|
3
2
|
import subprocess
|
|
4
3
|
import sys
|
|
@@ -13,7 +12,7 @@ from click import Context
|
|
|
13
12
|
|
|
14
13
|
from tinybird.tb.client import TinyB
|
|
15
14
|
from tinybird.tb.modules.cli import CLIException, cli
|
|
16
|
-
from tinybird.tb.modules.common import
|
|
15
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
|
|
17
16
|
from tinybird.tb.modules.config import CLIConfig
|
|
18
17
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
19
18
|
|
|
@@ -225,18 +224,18 @@ class Infrastructure:
|
|
|
225
224
|
def __init__(self, client: TinyB):
|
|
226
225
|
self.client = client
|
|
227
226
|
|
|
228
|
-
|
|
229
|
-
self.orgs =
|
|
227
|
+
def get_organizations_info(self, config: CLIConfig):
|
|
228
|
+
self.orgs = get_organizations_by_user(config)
|
|
230
229
|
|
|
231
|
-
|
|
230
|
+
def create_infra(self, name: str, host: str, organization_id: str) -> Dict[str, Any]:
|
|
232
231
|
"""Create a new infrastructure."""
|
|
233
|
-
infra =
|
|
232
|
+
infra = self.client.infra_create(organization_id=organization_id, name=name, host=host)
|
|
234
233
|
return infra
|
|
235
234
|
|
|
236
|
-
|
|
235
|
+
def list_infras(self) -> None:
|
|
237
236
|
"""List all self-managed regions for the admin organization."""
|
|
238
237
|
|
|
239
|
-
infras =
|
|
238
|
+
infras = self.get_infra_list()
|
|
240
239
|
columns = [
|
|
241
240
|
"name",
|
|
242
241
|
"host",
|
|
@@ -257,24 +256,24 @@ class Infrastructure:
|
|
|
257
256
|
echo_safe_humanfriendly_tables_format_smart_table(table_human_readable, column_names=columns)
|
|
258
257
|
click.echo("\n")
|
|
259
258
|
|
|
260
|
-
|
|
259
|
+
def remove_infra(self, name: str):
|
|
261
260
|
try:
|
|
262
261
|
click.echo(FeedbackManager.highlight(message=f"» Deleting infrastructure '{name}' from Tinybird..."))
|
|
263
|
-
infras =
|
|
262
|
+
infras = self.get_infra_list()
|
|
264
263
|
infra = next((infra for infra in infras if infra["name"] == name), None)
|
|
265
264
|
if not infra:
|
|
266
265
|
raise CLIException(f"Infrastructure '{name}' not found")
|
|
267
|
-
|
|
266
|
+
self.client.infra_delete(infra["id"], infra["organization_id"])
|
|
268
267
|
click.echo(FeedbackManager.success(message=f"\n✓ Infrastructure '{name}' deleted"))
|
|
269
268
|
except Exception as e:
|
|
270
269
|
click.echo(FeedbackManager.error(message=f"✗ Error: {e}"))
|
|
271
270
|
|
|
272
|
-
|
|
273
|
-
infras_list =
|
|
271
|
+
def update(self, infra_name: str, name: str, host: str):
|
|
272
|
+
infras_list = self.get_infra_list()
|
|
274
273
|
infra = next((infra for infra in infras_list if infra["name"] == infra_name), None)
|
|
275
274
|
if not infra:
|
|
276
275
|
return None
|
|
277
|
-
|
|
276
|
+
self.client.infra_update(
|
|
278
277
|
infra_id=infra["id"],
|
|
279
278
|
organization_id=infra["organization_id"],
|
|
280
279
|
name=name,
|
|
@@ -282,7 +281,7 @@ class Infrastructure:
|
|
|
282
281
|
)
|
|
283
282
|
return infra
|
|
284
283
|
|
|
285
|
-
|
|
284
|
+
def get_infra_list(self) -> List[Dict[str, str]]:
|
|
286
285
|
"""Get a list of all infrastructures across organizations.
|
|
287
286
|
|
|
288
287
|
Returns:
|
|
@@ -298,7 +297,7 @@ class Infrastructure:
|
|
|
298
297
|
org_id = org.get("id") or ""
|
|
299
298
|
org_name = org.get("name")
|
|
300
299
|
try:
|
|
301
|
-
infras =
|
|
300
|
+
infras = self.client.infra_list(organization_id=org_id)
|
|
302
301
|
for infra in infras:
|
|
303
302
|
infra["organization"] = org_name
|
|
304
303
|
all_infras.append(infra)
|
|
@@ -322,7 +321,7 @@ def infra(ctx: Context) -> None:
|
|
|
322
321
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
323
322
|
config = CLIConfig.get_project_config()
|
|
324
323
|
infra = Infrastructure(client)
|
|
325
|
-
|
|
324
|
+
infra.get_organizations_info(config)
|
|
326
325
|
ctx.ensure_object(dict)["infra"] = infra
|
|
327
326
|
|
|
328
327
|
|
|
@@ -340,8 +339,7 @@ def infra(ctx: Context) -> None:
|
|
|
340
339
|
@click.option("--skip-apply", is_flag=True, help="Skip Terraform and kubectl configuration and application")
|
|
341
340
|
@click.option("--organization-id", type=str, help="Organization ID for the self-managed region")
|
|
342
341
|
@click.pass_context
|
|
343
|
-
|
|
344
|
-
async def infra_init(
|
|
342
|
+
def infra_init(
|
|
345
343
|
ctx: Context,
|
|
346
344
|
name: str,
|
|
347
345
|
cloud_provider: str,
|
|
@@ -380,7 +378,7 @@ async def infra_init(
|
|
|
380
378
|
config = {}
|
|
381
379
|
if config_path.exists():
|
|
382
380
|
try:
|
|
383
|
-
with open(config_path, "r") as f:
|
|
381
|
+
with open(config_path, "r") as f:
|
|
384
382
|
config = json.load(f)
|
|
385
383
|
click.echo(FeedbackManager.info(message="** Loaded existing configuration from config.json"))
|
|
386
384
|
except json.JSONDecodeError:
|
|
@@ -442,7 +440,7 @@ async def infra_init(
|
|
|
442
440
|
"kubernetes_context": kubernetes_context,
|
|
443
441
|
}
|
|
444
442
|
|
|
445
|
-
with open(config_path, "w") as f:
|
|
443
|
+
with open(config_path, "w") as f:
|
|
446
444
|
json.dump(config, f, indent=2)
|
|
447
445
|
|
|
448
446
|
click.echo(FeedbackManager.info(message=f"** Configuration saved to {config_path}"))
|
|
@@ -450,16 +448,16 @@ async def infra_init(
|
|
|
450
448
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
451
449
|
cli_config = CLIConfig.get_project_config()
|
|
452
450
|
|
|
453
|
-
infras =
|
|
451
|
+
infras = infra.get_infra_list()
|
|
454
452
|
infra_name = next((infra for infra in infras if infra["name"] == name), None)
|
|
455
453
|
if not infra_name:
|
|
456
454
|
# Handle organization if not provided
|
|
457
|
-
organization_id, organization_name =
|
|
455
|
+
organization_id, organization_name = ask_for_organization(infra.orgs, organization_id)
|
|
458
456
|
if not organization_id:
|
|
459
457
|
return
|
|
460
458
|
click.echo(FeedbackManager.highlight(message=f"\n» Creating infrastructure '{name}' in Tinybird..."))
|
|
461
459
|
host = f"https://{dns_record}.{dns_zone_name}"
|
|
462
|
-
infra_obj =
|
|
460
|
+
infra_obj = infra.create_infra(name, host, organization_id)
|
|
463
461
|
else:
|
|
464
462
|
click.echo(FeedbackManager.highlight(message=f"» Infrastructure '{name}' already exists."))
|
|
465
463
|
if infra_name["host"] != f"https://{dns_record}.{dns_zone_name}":
|
|
@@ -469,7 +467,7 @@ async def infra_init(
|
|
|
469
467
|
)
|
|
470
468
|
)
|
|
471
469
|
if click.confirm("Would you like to update the host in the infra provisioned at Tinybird?"):
|
|
472
|
-
|
|
470
|
+
infra.update(infra_name=name, name=name, host=f"https://{dns_record}.{dns_zone_name}")
|
|
473
471
|
organization_name = infra_name["organization"]
|
|
474
472
|
infra_obj = infra_name
|
|
475
473
|
|
|
@@ -480,7 +478,7 @@ async def infra_init(
|
|
|
480
478
|
"dns_record": dns_record,
|
|
481
479
|
}
|
|
482
480
|
|
|
483
|
-
with open(tf_path, "w") as f:
|
|
481
|
+
with open(tf_path, "w") as f:
|
|
484
482
|
f.write(terraform_content.lstrip())
|
|
485
483
|
|
|
486
484
|
click.echo(FeedbackManager.info(message=f"** Created Terraform configuration in {tf_path}"))
|
|
@@ -498,7 +496,7 @@ async def infra_init(
|
|
|
498
496
|
"infra_user": cli_config.get_user_email() or "",
|
|
499
497
|
}
|
|
500
498
|
|
|
501
|
-
with open(yaml_path, "w") as f:
|
|
499
|
+
with open(yaml_path, "w") as f:
|
|
502
500
|
f.write(new_content.lstrip())
|
|
503
501
|
|
|
504
502
|
click.echo(FeedbackManager.info(message=f"** Created Kubernetes configuration in {yaml_path}"))
|
|
@@ -510,7 +508,7 @@ async def infra_init(
|
|
|
510
508
|
command = f"terraform -chdir={infra_dir} init"
|
|
511
509
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
512
510
|
|
|
513
|
-
init_result = subprocess.run(["terraform", f"-chdir={infra_dir}", "init"], capture_output=True, text=True)
|
|
511
|
+
init_result = subprocess.run(["terraform", f"-chdir={infra_dir}", "init"], capture_output=True, text=True)
|
|
514
512
|
|
|
515
513
|
if init_result.returncode != 0:
|
|
516
514
|
click.echo(FeedbackManager.error(message="✗ Error: Terraform initialization failed:"))
|
|
@@ -521,7 +519,7 @@ async def infra_init(
|
|
|
521
519
|
click.echo(FeedbackManager.info(message="** Running Terraform plan..."))
|
|
522
520
|
command = f"terraform -chdir={infra_dir} plan"
|
|
523
521
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
524
|
-
plan_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
522
|
+
plan_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
525
523
|
|
|
526
524
|
if plan_result.returncode != 0:
|
|
527
525
|
click.echo(FeedbackManager.error(message="✗ Error: Terraform plan failed:"))
|
|
@@ -536,7 +534,7 @@ async def infra_init(
|
|
|
536
534
|
click.echo(FeedbackManager.info(message="** Applying Terraform configuration..."))
|
|
537
535
|
command = f"terraform -chdir={infra_dir} apply -auto-approve"
|
|
538
536
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
539
|
-
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
537
|
+
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
540
538
|
|
|
541
539
|
if apply_result.returncode != 0:
|
|
542
540
|
click.echo(FeedbackManager.error(message="✗ Error: Terraform apply failed:"))
|
|
@@ -553,7 +551,7 @@ async def infra_init(
|
|
|
553
551
|
# Get current kubectl context
|
|
554
552
|
command = "kubectl config current-context"
|
|
555
553
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
556
|
-
current_context_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
554
|
+
current_context_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
557
555
|
|
|
558
556
|
current_context = (
|
|
559
557
|
current_context_result.stdout.strip() if current_context_result.returncode == 0 else "unknown"
|
|
@@ -563,7 +561,7 @@ async def infra_init(
|
|
|
563
561
|
command = "kubectl config get-contexts -o name"
|
|
564
562
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
565
563
|
|
|
566
|
-
contexts_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
564
|
+
contexts_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
567
565
|
|
|
568
566
|
if contexts_result.returncode != 0:
|
|
569
567
|
click.echo(FeedbackManager.error(message="✗ Error: Failed to get kubectl contexts:"))
|
|
@@ -605,7 +603,7 @@ async def infra_init(
|
|
|
605
603
|
|
|
606
604
|
# Update the config with the selected context
|
|
607
605
|
config["kubernetes_context"] = selected_context
|
|
608
|
-
with open(config_path, "w") as f:
|
|
606
|
+
with open(config_path, "w") as f:
|
|
609
607
|
json.dump(config, f, indent=2)
|
|
610
608
|
click.echo(f"Updated configuration with selected Kubernetes context: {selected_context}")
|
|
611
609
|
|
|
@@ -616,7 +614,7 @@ async def infra_init(
|
|
|
616
614
|
command = f"kubectl --context {selected_context} diff -f {str(yaml_path)}"
|
|
617
615
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
618
616
|
|
|
619
|
-
diff_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
617
|
+
diff_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
620
618
|
|
|
621
619
|
if diff_result.returncode not in [0, 1]: # kubectl diff returns 1 when there are differences
|
|
622
620
|
if (
|
|
@@ -638,7 +636,7 @@ async def infra_init(
|
|
|
638
636
|
# Now apply the configuration
|
|
639
637
|
command = f"kubectl --context {selected_context} apply -f {str(yaml_path)}"
|
|
640
638
|
click.echo(FeedbackManager.highlight(message=f"» Executing: {command}"))
|
|
641
|
-
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
639
|
+
apply_result = subprocess.run(command.split(), capture_output=True, text=True)
|
|
642
640
|
|
|
643
641
|
if apply_result.returncode != 0:
|
|
644
642
|
click.echo(FeedbackManager.error(message="✗ Error: Failed to apply Kubernetes configuration:"))
|
|
@@ -662,7 +660,7 @@ async def infra_init(
|
|
|
662
660
|
)
|
|
663
661
|
|
|
664
662
|
try:
|
|
665
|
-
response = requests.get(endpoint_url, allow_redirects=False, timeout=5)
|
|
663
|
+
response = requests.get(endpoint_url, allow_redirects=False, timeout=5)
|
|
666
664
|
if response.status_code < 400: # Consider any non-error response as success
|
|
667
665
|
click.echo(
|
|
668
666
|
"\n" + click.style("✅ HTTPS endpoint is now accessible!", fg="green", bold=True)
|
|
@@ -679,7 +677,7 @@ async def infra_init(
|
|
|
679
677
|
click.echo(" This might be due to DNS propagation or the Load Balancer provisioning delays")
|
|
680
678
|
click.echo(f" Try accessing {endpoint_url} manually in a few minutes")
|
|
681
679
|
else:
|
|
682
|
-
time.sleep(10)
|
|
680
|
+
time.sleep(10)
|
|
683
681
|
|
|
684
682
|
if not skip_apply:
|
|
685
683
|
# Print a summary with the endpoint URL
|
|
@@ -698,20 +696,18 @@ async def infra_init(
|
|
|
698
696
|
@infra.command(name="rm")
|
|
699
697
|
@click.argument("name")
|
|
700
698
|
@click.pass_context
|
|
701
|
-
|
|
702
|
-
async def infra_remove(ctx: click.Context, name: str):
|
|
699
|
+
def infra_remove(ctx: click.Context, name: str):
|
|
703
700
|
"""Delete an infrastructure from Tinybird"""
|
|
704
701
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
705
|
-
|
|
702
|
+
infra.remove_infra(name)
|
|
706
703
|
|
|
707
704
|
|
|
708
705
|
@infra.command(name="ls")
|
|
709
706
|
@click.pass_context
|
|
710
|
-
|
|
711
|
-
async def infra_list(ctx: click.Context):
|
|
707
|
+
def infra_list(ctx: click.Context):
|
|
712
708
|
"""List self-managed infrastructures"""
|
|
713
709
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
714
|
-
|
|
710
|
+
infra.list_infras()
|
|
715
711
|
|
|
716
712
|
|
|
717
713
|
@infra.command(name="add")
|
|
@@ -723,7 +719,7 @@ def infra_add(ctx: click.Context, name: str, host: Optional[str] = None, organiz
|
|
|
723
719
|
"""Creates a new self-managed region from an existing infrastructure URL."""
|
|
724
720
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
725
721
|
|
|
726
|
-
organization_id, organization_name =
|
|
722
|
+
organization_id, organization_name = ask_for_organization(infra.orgs, organization_id)
|
|
727
723
|
if not organization_id:
|
|
728
724
|
return
|
|
729
725
|
|
|
@@ -736,7 +732,7 @@ def infra_add(ctx: click.Context, name: str, host: Optional[str] = None, organiz
|
|
|
736
732
|
)
|
|
737
733
|
|
|
738
734
|
click.echo(FeedbackManager.highlight(message=f"» Adding self-managed region '{name}' in Tinybird..."))
|
|
739
|
-
new_infra =
|
|
735
|
+
new_infra = infra.create_infra(name, host, organization_id)
|
|
740
736
|
infra_token = new_infra["token"]
|
|
741
737
|
click.echo(
|
|
742
738
|
FeedbackManager.success(message=f"\n✓ Self-managed region '{name}' added in '{organization_name}' Organization")
|
|
@@ -758,8 +754,7 @@ def infra_add(ctx: click.Context, name: str, host: Optional[str] = None, organiz
|
|
|
758
754
|
@click.option("--name", type=str, help="Name for identifying the self-managed region in Tinybird")
|
|
759
755
|
@click.option("--host", type=str, help="Host for the self-managed region")
|
|
760
756
|
@click.pass_context
|
|
761
|
-
|
|
762
|
-
async def infra_update(ctx: click.Context, infra_name: str, name: str, host: str):
|
|
757
|
+
def infra_update(ctx: click.Context, infra_name: str, name: str, host: str):
|
|
763
758
|
"""Updates the URL of an existing self-managed region."""
|
|
764
759
|
infra: Infrastructure = ctx.ensure_object(dict)["infra"]
|
|
765
760
|
if not name and not host:
|
|
@@ -771,7 +766,7 @@ async def infra_update(ctx: click.Context, infra_name: str, name: str, host: str
|
|
|
771
766
|
click.echo(
|
|
772
767
|
FeedbackManager.highlight(message=f"» Updating self-managed region'{infra_name}' in Tinybird...")
|
|
773
768
|
)
|
|
774
|
-
infra_id =
|
|
769
|
+
infra_id = infra.update(infra_name, name, host)
|
|
775
770
|
if not infra_id:
|
|
776
771
|
raise CLIException(f"Self-managed region '{infra_name}' not found")
|
|
777
772
|
except Exception as e:
|
tinybird/tb/modules/job.py
CHANGED
|
@@ -10,7 +10,7 @@ from click import Context
|
|
|
10
10
|
|
|
11
11
|
from tinybird.tb.client import DoesNotExistException, TinyB
|
|
12
12
|
from tinybird.tb.modules.cli import cli
|
|
13
|
-
from tinybird.tb.modules.common import
|
|
13
|
+
from tinybird.tb.modules.common import echo_safe_humanfriendly_tables_format_smart_table
|
|
14
14
|
from tinybird.tb.modules.exceptions import CLIException
|
|
15
15
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
16
16
|
|
|
@@ -38,11 +38,10 @@ def job(ctx: Context) -> None:
|
|
|
38
38
|
default=None,
|
|
39
39
|
)
|
|
40
40
|
@click.pass_context
|
|
41
|
-
|
|
42
|
-
async def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...]) -> None:
|
|
41
|
+
def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...]) -> None:
|
|
43
42
|
"""List jobs, up to 100"""
|
|
44
43
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
45
|
-
jobs =
|
|
44
|
+
jobs = client.jobs(status=status, kind=kind)
|
|
46
45
|
columns = ["id", "kind", "status", "created at", "updated at", "job url"]
|
|
47
46
|
click.echo(FeedbackManager.info_jobs())
|
|
48
47
|
table = []
|
|
@@ -55,11 +54,10 @@ async def jobs_ls(ctx: Context, status: Tuple[str, ...], kind: Tuple[str, ...])
|
|
|
55
54
|
@job.command(name="details")
|
|
56
55
|
@click.argument("job_id")
|
|
57
56
|
@click.pass_context
|
|
58
|
-
|
|
59
|
-
async def job_details(ctx: Context, job_id: str) -> None:
|
|
57
|
+
def job_details(ctx: Context, job_id: str) -> None:
|
|
60
58
|
"""Get details for any job created in the last 48h"""
|
|
61
59
|
client: TinyB = ctx.ensure_object(dict)["client"]
|
|
62
|
-
job =
|
|
60
|
+
job = client.job(job_id)
|
|
63
61
|
columns = []
|
|
64
62
|
click.echo(FeedbackManager.info_job(job=job_id))
|
|
65
63
|
table = []
|
|
@@ -72,13 +70,12 @@ async def job_details(ctx: Context, job_id: str) -> None:
|
|
|
72
70
|
@job.command(name="cancel")
|
|
73
71
|
@click.argument("job_id")
|
|
74
72
|
@click.pass_context
|
|
75
|
-
|
|
76
|
-
async def job_cancel(ctx: Context, job_id: str) -> None:
|
|
73
|
+
def job_cancel(ctx: Context, job_id: str) -> None:
|
|
77
74
|
"""Try to cancel a job"""
|
|
78
75
|
client = ctx.ensure_object(dict)["client"]
|
|
79
76
|
|
|
80
77
|
try:
|
|
81
|
-
result =
|
|
78
|
+
result = client.job_cancel(job_id)
|
|
82
79
|
except DoesNotExistException:
|
|
83
80
|
raise CLIException(FeedbackManager.error_job_does_not_exist(job_id=job_id))
|
|
84
81
|
except Exception as e:
|
tinybird/tb/modules/local.py
CHANGED
|
@@ -5,7 +5,6 @@ import requests
|
|
|
5
5
|
|
|
6
6
|
from docker.client import DockerClient
|
|
7
7
|
from tinybird.tb.modules.cli import cli
|
|
8
|
-
from tinybird.tb.modules.common import coro
|
|
9
8
|
from tinybird.tb.modules.exceptions import CLIException
|
|
10
9
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
11
10
|
from tinybird.tb.modules.local_common import (
|
|
@@ -26,11 +25,11 @@ def stop_tinybird_local(docker_client: DockerClient) -> None:
|
|
|
26
25
|
pass
|
|
27
26
|
|
|
28
27
|
|
|
29
|
-
def remove_tinybird_local(docker_client: DockerClient) -> None:
|
|
28
|
+
def remove_tinybird_local(docker_client: DockerClient, persist_data: bool) -> None:
|
|
30
29
|
"""Remove the Tinybird container."""
|
|
31
30
|
try:
|
|
32
31
|
container = docker_client.containers.get(TB_CONTAINER_NAME)
|
|
33
|
-
if click.confirm(
|
|
32
|
+
if persist_data or click.confirm(
|
|
34
33
|
FeedbackManager.warning(
|
|
35
34
|
message="△ This step will remove all your data inside Tinybird Local. Are you sure? [y/N]:"
|
|
36
35
|
),
|
|
@@ -88,8 +87,7 @@ def local(ctx: click.Context) -> None:
|
|
|
88
87
|
|
|
89
88
|
|
|
90
89
|
@local.command()
|
|
91
|
-
|
|
92
|
-
async def stop() -> None:
|
|
90
|
+
def stop() -> None:
|
|
93
91
|
"""Stop Tinybird Local"""
|
|
94
92
|
click.echo(FeedbackManager.highlight(message="» Shutting down Tinybird Local..."))
|
|
95
93
|
docker_client = get_docker_client()
|
|
@@ -99,8 +97,7 @@ async def stop() -> None:
|
|
|
99
97
|
|
|
100
98
|
@local.command()
|
|
101
99
|
@click.pass_context
|
|
102
|
-
|
|
103
|
-
async def status(ctx: click.Context) -> None:
|
|
100
|
+
def status(ctx: click.Context) -> None:
|
|
104
101
|
"""Check status of Tinybird Local"""
|
|
105
102
|
docker_client = get_docker_client()
|
|
106
103
|
container = get_existing_container_with_matching_env(docker_client, TB_CONTAINER_NAME, {})
|
|
@@ -115,7 +112,7 @@ async def status(ctx: click.Context) -> None:
|
|
|
115
112
|
from tinybird.tb.modules.info import get_local_info
|
|
116
113
|
|
|
117
114
|
config = ctx.ensure_object(dict).get("config", {})
|
|
118
|
-
|
|
115
|
+
get_local_info(config)
|
|
119
116
|
elif status == "restarting" or (status == "running" and health == "starting"):
|
|
120
117
|
click.echo(FeedbackManager.highlight(message="* Tinybird Local is starting..."))
|
|
121
118
|
elif status == "removing":
|
|
@@ -129,46 +126,53 @@ async def status(ctx: click.Context) -> None:
|
|
|
129
126
|
|
|
130
127
|
|
|
131
128
|
@local.command()
|
|
132
|
-
|
|
133
|
-
async def remove() -> None:
|
|
129
|
+
def remove() -> None:
|
|
134
130
|
"""Remove Tinybird Local"""
|
|
135
131
|
click.echo(FeedbackManager.highlight(message="» Removing Tinybird Local..."))
|
|
136
132
|
docker_client = get_docker_client()
|
|
137
|
-
remove_tinybird_local(docker_client)
|
|
133
|
+
remove_tinybird_local(docker_client, persist_data=False)
|
|
138
134
|
click.echo(FeedbackManager.success(message="✓ Tinybird Local removed"))
|
|
139
135
|
|
|
140
136
|
|
|
141
137
|
@local.command()
|
|
142
|
-
@coro
|
|
143
138
|
@click.option(
|
|
144
139
|
"--use-aws-creds",
|
|
145
140
|
default=False,
|
|
146
141
|
is_flag=True,
|
|
147
142
|
help="Use local AWS credentials from your environment and pass them to the Tinybird docker container",
|
|
148
143
|
)
|
|
149
|
-
|
|
144
|
+
@click.option(
|
|
145
|
+
"--volumes-path",
|
|
146
|
+
default=None,
|
|
147
|
+
help="Path to the volumes directory. If not provided, the container data won't be persisted.",
|
|
148
|
+
)
|
|
149
|
+
def start(use_aws_creds: bool, volumes_path: str) -> None:
|
|
150
150
|
"""Start Tinybird Local"""
|
|
151
151
|
click.echo(FeedbackManager.highlight(message="» Starting Tinybird Local..."))
|
|
152
152
|
docker_client = get_docker_client()
|
|
153
|
-
start_tinybird_local(docker_client, use_aws_creds)
|
|
153
|
+
start_tinybird_local(docker_client, use_aws_creds, volumes_path)
|
|
154
154
|
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
155
155
|
|
|
156
156
|
|
|
157
157
|
@local.command()
|
|
158
|
-
@coro
|
|
159
158
|
@click.option(
|
|
160
159
|
"--use-aws-creds",
|
|
161
160
|
default=False,
|
|
162
161
|
is_flag=True,
|
|
163
162
|
help="Use local AWS credentials from your environment and pass them to the Tinybird docker container",
|
|
164
163
|
)
|
|
165
|
-
|
|
164
|
+
@click.option(
|
|
165
|
+
"--volumes-path",
|
|
166
|
+
default=None,
|
|
167
|
+
help="Path to the volumes directory. If not provided, the container data won't be persisted.",
|
|
168
|
+
)
|
|
169
|
+
def restart(use_aws_creds: bool, volumes_path: str) -> None:
|
|
166
170
|
"""Restart Tinybird Local"""
|
|
167
171
|
click.echo(FeedbackManager.highlight(message="» Restarting Tinybird Local..."))
|
|
168
172
|
docker_client = get_docker_client()
|
|
169
|
-
remove_tinybird_local(docker_client)
|
|
173
|
+
remove_tinybird_local(docker_client, volumes_path is not None)
|
|
170
174
|
click.echo(FeedbackManager.info(message="✓ Tinybird Local stopped"))
|
|
171
|
-
start_tinybird_local(docker_client, use_aws_creds)
|
|
175
|
+
start_tinybird_local(docker_client, use_aws_creds, volumes_path)
|
|
172
176
|
click.echo(FeedbackManager.success(message="✓ Tinybird Local is ready!"))
|
|
173
177
|
|
|
174
178
|
|
|
@@ -28,16 +28,16 @@ TB_LOCAL_ADDRESS = f"http://{TB_LOCAL_HOST}:{TB_LOCAL_PORT}"
|
|
|
28
28
|
TB_LOCAL_DEFAULT_WORKSPACE_NAME = "Tinybird_Local_Testing"
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
def get_tinybird_local_client(
|
|
32
32
|
config_obj: Dict[str, Any], test: bool = False, staging: bool = False, silent: bool = False
|
|
33
33
|
) -> TinyB:
|
|
34
34
|
"""Get a Tinybird client connected to the local environment."""
|
|
35
35
|
|
|
36
|
-
config =
|
|
36
|
+
config = get_tinybird_local_config(config_obj, test=test, silent=silent)
|
|
37
37
|
return config.get_client(host=TB_LOCAL_ADDRESS, staging=staging)
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = False, silent: bool = False) -> CLIConfig:
|
|
41
41
|
"""Craft a client config with a workspace name based on the path of the project files
|
|
42
42
|
|
|
43
43
|
It uses the tokens from tinybird local
|
|
@@ -76,7 +76,7 @@ async def get_tinybird_local_config(config_obj: Dict[str, Any], test: bool = Fal
|
|
|
76
76
|
ws = None
|
|
77
77
|
|
|
78
78
|
if not ws:
|
|
79
|
-
|
|
79
|
+
user_client.create_workspace(ws_name, assign_to_organization_id=user_org_id, version="v1")
|
|
80
80
|
user_workspaces = requests.get(f"{TB_LOCAL_ADDRESS}/v1/user/workspaces?token={admin_token}").json()
|
|
81
81
|
ws = next((ws for ws in user_workspaces["workspaces"] if ws["name"] == ws_name), None)
|
|
82
82
|
if not ws:
|
|
@@ -217,6 +217,7 @@ def get_local_tokens() -> Dict[str, str]:
|
|
|
217
217
|
def start_tinybird_local(
|
|
218
218
|
docker_client: DockerClient,
|
|
219
219
|
use_aws_creds: bool,
|
|
220
|
+
volumes_path: Optional[str] = None,
|
|
220
221
|
) -> None:
|
|
221
222
|
"""Start the Tinybird container."""
|
|
222
223
|
pull_show_prompt = False
|
|
@@ -252,6 +253,13 @@ def start_tinybird_local(
|
|
|
252
253
|
if container:
|
|
253
254
|
container.remove(force=True)
|
|
254
255
|
|
|
256
|
+
volumes = {}
|
|
257
|
+
if volumes_path:
|
|
258
|
+
volumes = {
|
|
259
|
+
f"{volumes_path}/data": {"bind": "/var/lib/clickhouse", "mode": "rw"},
|
|
260
|
+
f"{volumes_path}/metadata": {"bind": "/redis-data", "mode": "rw"},
|
|
261
|
+
}
|
|
262
|
+
|
|
255
263
|
container = docker_client.containers.run(
|
|
256
264
|
TB_IMAGE_NAME,
|
|
257
265
|
name=TB_CONTAINER_NAME,
|
|
@@ -260,6 +268,7 @@ def start_tinybird_local(
|
|
|
260
268
|
remove=False,
|
|
261
269
|
platform="linux/amd64",
|
|
262
270
|
environment=environment,
|
|
271
|
+
volumes=volumes,
|
|
263
272
|
)
|
|
264
273
|
|
|
265
274
|
click.echo(FeedbackManager.info(message="* Waiting for Tinybird Local to be ready..."))
|
tinybird/tb/modules/login.py
CHANGED
|
@@ -19,7 +19,7 @@ import requests
|
|
|
19
19
|
|
|
20
20
|
from tinybird.tb.config import DEFAULT_API_HOST
|
|
21
21
|
from tinybird.tb.modules.cli import CLIConfig, cli
|
|
22
|
-
from tinybird.tb.modules.common import ask_for_region_interactively,
|
|
22
|
+
from tinybird.tb.modules.common import ask_for_region_interactively, get_regions
|
|
23
23
|
from tinybird.tb.modules.exceptions import CLILoginException
|
|
24
24
|
from tinybird.tb.modules.feedback_manager import FeedbackManager
|
|
25
25
|
|
|
@@ -135,8 +135,7 @@ def start_server(auth_callback, auth_host):
|
|
|
135
135
|
default="browser",
|
|
136
136
|
help="Set the authentication method to use. Default: browser.",
|
|
137
137
|
)
|
|
138
|
-
|
|
139
|
-
async def login(host: Optional[str], auth_host: str, workspace: str, interactive: bool, method: str):
|
|
138
|
+
def login(host: Optional[str], auth_host: str, workspace: str, interactive: bool, method: str):
|
|
140
139
|
"""Authenticate using the browser."""
|
|
141
140
|
try:
|
|
142
141
|
cli_config = CLIConfig.get_project_config()
|
|
@@ -148,7 +147,7 @@ async def login(host: Optional[str], auth_host: str, workspace: str, interactive
|
|
|
148
147
|
else:
|
|
149
148
|
click.echo(FeedbackManager.highlight(message="» No region detected, select one from the list below:"))
|
|
150
149
|
|
|
151
|
-
regions =
|
|
150
|
+
regions = get_regions(cli_config)
|
|
152
151
|
selected_region = ask_for_region_interactively(regions)
|
|
153
152
|
|
|
154
153
|
# If the user cancels the selection, we'll exit
|
|
@@ -202,7 +201,7 @@ async def login(host: Optional[str], auth_host: str, workspace: str, interactive
|
|
|
202
201
|
|
|
203
202
|
time.sleep(2) # noqa: ASYNC251
|
|
204
203
|
|
|
205
|
-
|
|
204
|
+
poll_for_tokens()
|
|
206
205
|
return
|
|
207
206
|
|
|
208
207
|
auth_event = threading.Event()
|
|
@@ -236,12 +235,12 @@ async def login(host: Optional[str], auth_host: str, workspace: str, interactive
|
|
|
236
235
|
if auth_event.wait(timeout=SERVER_MAX_WAIT_TIME): # Wait for up to 180 seconds
|
|
237
236
|
params = {}
|
|
238
237
|
params["code"] = auth_code[0]
|
|
239
|
-
response = requests.get(
|
|
238
|
+
response = requests.get(
|
|
240
239
|
f"{auth_host}/api/cli-login?{urlencode(params)}",
|
|
241
240
|
)
|
|
242
241
|
|
|
243
242
|
data = response.json()
|
|
244
|
-
|
|
243
|
+
authenticate_with_tokens(data, host, cli_config)
|
|
245
244
|
else:
|
|
246
245
|
raise Exception("Authentication failed or timed out.")
|
|
247
246
|
except Exception as e:
|
|
@@ -308,13 +307,13 @@ def create_one_time_code():
|
|
|
308
307
|
return seperator.join(parts), full_code
|
|
309
308
|
|
|
310
309
|
|
|
311
|
-
|
|
310
|
+
def authenticate_with_tokens(data: Dict[str, Any], host: Optional[str], cli_config: CLIConfig):
|
|
312
311
|
cli_config.set_token(data.get("workspace_token", ""))
|
|
313
312
|
host = host or data.get("api_host", "")
|
|
314
313
|
cli_config.set_token_for_host(data.get("workspace_token", ""), host)
|
|
315
314
|
cli_config.set_user_token(data.get("user_token", ""))
|
|
316
315
|
cli_config.set_host(host)
|
|
317
|
-
ws =
|
|
316
|
+
ws = cli_config.get_client(token=data.get("workspace_token", ""), host=host).workspace_info(version="v1")
|
|
318
317
|
for k in ("id", "name", "user_email", "user_id", "scope"):
|
|
319
318
|
if k in ws:
|
|
320
319
|
cli_config[k] = ws[k]
|
|
@@ -322,7 +321,7 @@ async def authenticate_with_tokens(data: Dict[str, Any], host: Optional[str], cl
|
|
|
322
321
|
path = os.path.join(os.getcwd(), ".tinyb")
|
|
323
322
|
cli_config.persist_to_file(override_with_path=path)
|
|
324
323
|
|
|
325
|
-
auth_info: Dict[str, Any] =
|
|
324
|
+
auth_info: Dict[str, Any] = cli_config.get_user_client().check_auth_login()
|
|
326
325
|
if not auth_info.get("is_valid", False):
|
|
327
326
|
raise Exception(FeedbackManager.error_auth_login_not_valid(host=cli_config.get_host()))
|
|
328
327
|
|