starrocks-br 0.5.2__py3-none-any.whl → 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- starrocks_br/cli.py +103 -36
- starrocks_br/concurrency.py +33 -23
- starrocks_br/config.py +75 -0
- starrocks_br/error_handler.py +59 -12
- starrocks_br/exceptions.py +14 -0
- starrocks_br/executor.py +9 -2
- starrocks_br/history.py +9 -9
- starrocks_br/labels.py +5 -3
- starrocks_br/planner.py +56 -13
- starrocks_br/repository.py +1 -1
- starrocks_br/restore.py +197 -40
- starrocks_br/schema.py +89 -43
- {starrocks_br-0.5.2.dist-info → starrocks_br-0.6.0.dist-info}/METADATA +15 -2
- starrocks_br-0.6.0.dist-info/RECORD +24 -0
- {starrocks_br-0.5.2.dist-info → starrocks_br-0.6.0.dist-info}/WHEEL +1 -1
- starrocks_br-0.5.2.dist-info/RECORD +0 -24
- {starrocks_br-0.5.2.dist-info → starrocks_br-0.6.0.dist-info}/entry_points.txt +0 -0
- {starrocks_br-0.5.2.dist-info → starrocks_br-0.6.0.dist-info}/licenses/LICENSE +0 -0
- {starrocks_br-0.5.2.dist-info → starrocks_br-0.6.0.dist-info}/top_level.txt +0 -0
starrocks_br/restore.py
CHANGED
|
@@ -201,9 +201,22 @@ def execute_restore(
|
|
|
201
201
|
max_polls: int = MAX_POLLS,
|
|
202
202
|
poll_interval: float = 1.0,
|
|
203
203
|
scope: str = "restore",
|
|
204
|
+
ops_database: str = "ops",
|
|
204
205
|
) -> dict:
|
|
205
206
|
"""Execute a complete restore workflow: submit command and monitor progress.
|
|
206
207
|
|
|
208
|
+
Args:
|
|
209
|
+
db: Database connection
|
|
210
|
+
restore_command: Restore SQL command to execute
|
|
211
|
+
backup_label: Label of the backup being restored
|
|
212
|
+
restore_type: Type of restore operation
|
|
213
|
+
repository: Repository name
|
|
214
|
+
database: Database name
|
|
215
|
+
max_polls: Maximum polling attempts
|
|
216
|
+
poll_interval: Seconds between polls
|
|
217
|
+
scope: Job scope (for concurrency control)
|
|
218
|
+
ops_database: Name of ops database (default: "ops")
|
|
219
|
+
|
|
207
220
|
Returns dictionary with keys: success, final_status, error_message
|
|
208
221
|
"""
|
|
209
222
|
cluster_tz = db.timezone
|
|
@@ -240,13 +253,18 @@ def execute_restore(
|
|
|
240
253
|
"finished_at": finished_at,
|
|
241
254
|
"error_message": None if success else final_status["state"],
|
|
242
255
|
},
|
|
256
|
+
ops_database=ops_database,
|
|
243
257
|
)
|
|
244
258
|
except Exception as e:
|
|
245
259
|
logger.error(f"Failed to log restore history: {str(e)}")
|
|
246
260
|
|
|
247
261
|
try:
|
|
248
262
|
concurrency.complete_job_slot(
|
|
249
|
-
db,
|
|
263
|
+
db,
|
|
264
|
+
scope=scope,
|
|
265
|
+
label=label,
|
|
266
|
+
final_state=final_status["state"],
|
|
267
|
+
ops_database=ops_database,
|
|
250
268
|
)
|
|
251
269
|
except Exception as e:
|
|
252
270
|
logger.error(f"Failed to complete job slot: {str(e)}")
|
|
@@ -264,7 +282,7 @@ def execute_restore(
|
|
|
264
282
|
return {"success": False, "final_status": None, "error_message": str(e)}
|
|
265
283
|
|
|
266
284
|
|
|
267
|
-
def find_restore_pair(db, target_label: str) -> list[str]:
|
|
285
|
+
def find_restore_pair(db, target_label: str, ops_database: str = "ops") -> list[str]:
|
|
268
286
|
"""Find the correct sequence of backups needed for restore.
|
|
269
287
|
|
|
270
288
|
Args:
|
|
@@ -280,7 +298,7 @@ def find_restore_pair(db, target_label: str) -> list[str]:
|
|
|
280
298
|
"""
|
|
281
299
|
query = f"""
|
|
282
300
|
SELECT label, backup_type, finished_at
|
|
283
|
-
FROM
|
|
301
|
+
FROM {ops_database}.backup_history
|
|
284
302
|
WHERE label = {utils.quote_value(target_label)}
|
|
285
303
|
AND status = 'FINISHED'
|
|
286
304
|
"""
|
|
@@ -299,7 +317,7 @@ def find_restore_pair(db, target_label: str) -> list[str]:
|
|
|
299
317
|
|
|
300
318
|
full_backup_query = f"""
|
|
301
319
|
SELECT label, backup_type, finished_at
|
|
302
|
-
FROM
|
|
320
|
+
FROM {ops_database}.backup_history
|
|
303
321
|
WHERE backup_type = 'full'
|
|
304
322
|
AND status = 'FINISHED'
|
|
305
323
|
AND label LIKE {utils.quote_value(f"{database_name}_%")}
|
|
@@ -326,6 +344,7 @@ def get_tables_from_backup(
|
|
|
326
344
|
group: str | None = None,
|
|
327
345
|
table: str | None = None,
|
|
328
346
|
database: str | None = None,
|
|
347
|
+
ops_database: str = "ops",
|
|
329
348
|
) -> list[str]:
|
|
330
349
|
"""Get list of tables to restore from backup manifest.
|
|
331
350
|
|
|
@@ -354,7 +373,7 @@ def get_tables_from_backup(
|
|
|
354
373
|
|
|
355
374
|
query = f"""
|
|
356
375
|
SELECT DISTINCT database_name, table_name
|
|
357
|
-
FROM
|
|
376
|
+
FROM {ops_database}.backup_partitions
|
|
358
377
|
WHERE label = {utils.quote_value(label)}
|
|
359
378
|
ORDER BY database_name, table_name
|
|
360
379
|
"""
|
|
@@ -377,7 +396,7 @@ def get_tables_from_backup(
|
|
|
377
396
|
if group:
|
|
378
397
|
group_query = f"""
|
|
379
398
|
SELECT database_name, table_name
|
|
380
|
-
FROM
|
|
399
|
+
FROM {ops_database}.table_inventory
|
|
381
400
|
WHERE inventory_group = {utils.quote_value(group)}
|
|
382
401
|
"""
|
|
383
402
|
|
|
@@ -404,6 +423,35 @@ def get_tables_from_backup(
|
|
|
404
423
|
return tables
|
|
405
424
|
|
|
406
425
|
|
|
426
|
+
def get_partitions_from_backup(
|
|
427
|
+
db, label: str, table: str, ops_database: str = "ops"
|
|
428
|
+
) -> list[str]:
|
|
429
|
+
"""Get list of partitions for a specific table from backup manifest.
|
|
430
|
+
|
|
431
|
+
Args:
|
|
432
|
+
db: Database connection
|
|
433
|
+
label: Backup label
|
|
434
|
+
table: Table name in format 'database.table'
|
|
435
|
+
ops_database: Operations database name
|
|
436
|
+
|
|
437
|
+
Returns:
|
|
438
|
+
List of partition names for the table in this backup
|
|
439
|
+
"""
|
|
440
|
+
database_name, table_name = table.split(".", 1)
|
|
441
|
+
|
|
442
|
+
query = f"""
|
|
443
|
+
SELECT partition_name
|
|
444
|
+
FROM {ops_database}.backup_partitions
|
|
445
|
+
WHERE label = {utils.quote_value(label)}
|
|
446
|
+
AND database_name = {utils.quote_value(database_name)}
|
|
447
|
+
AND table_name = {utils.quote_value(table_name)}
|
|
448
|
+
ORDER BY partition_name
|
|
449
|
+
"""
|
|
450
|
+
|
|
451
|
+
rows = db.query(query)
|
|
452
|
+
return [row[0] for row in rows]
|
|
453
|
+
|
|
454
|
+
|
|
407
455
|
def execute_restore_flow(
|
|
408
456
|
db,
|
|
409
457
|
repo_name: str,
|
|
@@ -411,6 +459,7 @@ def execute_restore_flow(
|
|
|
411
459
|
tables_to_restore: list[str],
|
|
412
460
|
rename_suffix: str = "_restored",
|
|
413
461
|
skip_confirmation: bool = False,
|
|
462
|
+
ops_database: str = "ops",
|
|
414
463
|
) -> dict:
|
|
415
464
|
"""Execute the complete restore flow with safety measures.
|
|
416
465
|
|
|
@@ -421,6 +470,7 @@ def execute_restore_flow(
|
|
|
421
470
|
tables_to_restore: List of tables to restore (format: database.table)
|
|
422
471
|
rename_suffix: Suffix for temporary tables
|
|
423
472
|
skip_confirmation: If True, skip interactive confirmation prompt
|
|
473
|
+
ops_database: Name of ops database (default: "ops")
|
|
424
474
|
|
|
425
475
|
Returns:
|
|
426
476
|
Dictionary with success status and details
|
|
@@ -452,59 +502,122 @@ def execute_restore_flow(
|
|
|
452
502
|
database_name = tables_to_restore[0].split(".")[0]
|
|
453
503
|
|
|
454
504
|
base_label = restore_pair[0]
|
|
455
|
-
logger.info("")
|
|
456
|
-
logger.info(f"Step 1: Restoring base backup '{base_label}'...")
|
|
457
|
-
|
|
458
|
-
base_timestamp = get_snapshot_timestamp(db, repo_name, base_label)
|
|
459
|
-
|
|
460
|
-
base_restore_command = _build_restore_command_with_rename(
|
|
461
|
-
base_label, repo_name, tables_to_restore, rename_suffix, database_name, base_timestamp
|
|
462
|
-
)
|
|
463
505
|
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
)
|
|
467
|
-
|
|
468
|
-
if not base_result["success"]:
|
|
469
|
-
return {
|
|
470
|
-
"success": False,
|
|
471
|
-
"error_message": f"Base restore failed: {base_result['error_message']}",
|
|
472
|
-
}
|
|
473
|
-
|
|
474
|
-
logger.success("Base restore completed successfully")
|
|
506
|
+
tables_in_base = get_tables_from_backup(db, base_label, ops_database=ops_database)
|
|
507
|
+
tables_to_restore_from_base = [t for t in tables_to_restore if t in tables_in_base]
|
|
475
508
|
|
|
476
|
-
if
|
|
477
|
-
incremental_label = restore_pair[1]
|
|
509
|
+
if tables_to_restore_from_base:
|
|
478
510
|
logger.info("")
|
|
479
|
-
logger.info(f"Step
|
|
511
|
+
logger.info(f"Step 1: Restoring base backup '{base_label}'...")
|
|
480
512
|
|
|
481
|
-
|
|
513
|
+
base_timestamp = get_snapshot_timestamp(db, repo_name, base_label)
|
|
482
514
|
|
|
483
|
-
|
|
484
|
-
|
|
515
|
+
base_restore_command = _build_restore_command_with_rename(
|
|
516
|
+
base_label,
|
|
485
517
|
repo_name,
|
|
486
|
-
|
|
518
|
+
tables_to_restore_from_base,
|
|
519
|
+
rename_suffix,
|
|
487
520
|
database_name,
|
|
488
|
-
|
|
521
|
+
base_timestamp,
|
|
489
522
|
)
|
|
490
523
|
|
|
491
|
-
|
|
524
|
+
base_result = execute_restore(
|
|
492
525
|
db,
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
"
|
|
526
|
+
base_restore_command,
|
|
527
|
+
base_label,
|
|
528
|
+
"full",
|
|
496
529
|
repo_name,
|
|
497
530
|
database_name,
|
|
498
531
|
scope="restore",
|
|
532
|
+
ops_database=ops_database,
|
|
499
533
|
)
|
|
500
534
|
|
|
501
|
-
if not
|
|
535
|
+
if not base_result["success"]:
|
|
502
536
|
return {
|
|
503
537
|
"success": False,
|
|
504
|
-
"error_message": f"
|
|
538
|
+
"error_message": f"Base restore failed: {base_result['error_message']}",
|
|
505
539
|
}
|
|
506
540
|
|
|
507
|
-
logger.success("
|
|
541
|
+
logger.success("Base restore completed successfully")
|
|
542
|
+
else:
|
|
543
|
+
logger.info("")
|
|
544
|
+
logger.info(
|
|
545
|
+
f"Step 1: Skipping base backup '{base_label}' (no requested tables in this backup)"
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
if len(restore_pair) > 1:
|
|
549
|
+
incremental_label = restore_pair[1]
|
|
550
|
+
|
|
551
|
+
tables_in_incremental = get_tables_from_backup(
|
|
552
|
+
db, incremental_label, ops_database=ops_database
|
|
553
|
+
)
|
|
554
|
+
tables_to_restore_from_incremental = [
|
|
555
|
+
t for t in tables_to_restore if t in tables_in_incremental
|
|
556
|
+
]
|
|
557
|
+
|
|
558
|
+
if not tables_to_restore_from_incremental:
|
|
559
|
+
logger.info("")
|
|
560
|
+
logger.info(
|
|
561
|
+
f"Step 2: Skipping incremental backup '{incremental_label}' (no requested tables in this backup)"
|
|
562
|
+
)
|
|
563
|
+
else:
|
|
564
|
+
logger.info("")
|
|
565
|
+
logger.info(f"Step 2: Applying incremental backup '{incremental_label}'...")
|
|
566
|
+
|
|
567
|
+
incremental_timestamp = get_snapshot_timestamp(db, repo_name, incremental_label)
|
|
568
|
+
|
|
569
|
+
for table in tables_to_restore_from_incremental:
|
|
570
|
+
partitions = get_partitions_from_backup(
|
|
571
|
+
db, incremental_label, table, ops_database=ops_database
|
|
572
|
+
)
|
|
573
|
+
|
|
574
|
+
if not partitions:
|
|
575
|
+
logger.warning(f"No partitions found for {table} in {incremental_label}, skipping")
|
|
576
|
+
continue
|
|
577
|
+
|
|
578
|
+
table_was_in_base = table in tables_to_restore_from_base
|
|
579
|
+
|
|
580
|
+
if table_was_in_base:
|
|
581
|
+
_, table_name = table.split(".", 1)
|
|
582
|
+
target_table_name = f"{table_name}{rename_suffix}"
|
|
583
|
+
incremental_restore_command = _build_partition_restore_command(
|
|
584
|
+
incremental_label,
|
|
585
|
+
repo_name,
|
|
586
|
+
f"{database_name}.{target_table_name}",
|
|
587
|
+
partitions,
|
|
588
|
+
database_name,
|
|
589
|
+
incremental_timestamp,
|
|
590
|
+
rename_suffix=None,
|
|
591
|
+
)
|
|
592
|
+
else:
|
|
593
|
+
incremental_restore_command = _build_partition_restore_command(
|
|
594
|
+
incremental_label,
|
|
595
|
+
repo_name,
|
|
596
|
+
table,
|
|
597
|
+
partitions,
|
|
598
|
+
database_name,
|
|
599
|
+
incremental_timestamp,
|
|
600
|
+
rename_suffix=rename_suffix,
|
|
601
|
+
)
|
|
602
|
+
|
|
603
|
+
incremental_result = execute_restore(
|
|
604
|
+
db,
|
|
605
|
+
incremental_restore_command,
|
|
606
|
+
incremental_label,
|
|
607
|
+
"incremental",
|
|
608
|
+
repo_name,
|
|
609
|
+
database_name,
|
|
610
|
+
scope="restore",
|
|
611
|
+
ops_database=ops_database,
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
if not incremental_result["success"]:
|
|
615
|
+
return {
|
|
616
|
+
"success": False,
|
|
617
|
+
"error_message": f"Incremental restore failed for {table}: {incremental_result['error_message']}",
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
logger.success("Incremental restore completed successfully")
|
|
508
621
|
|
|
509
622
|
logger.info("")
|
|
510
623
|
logger.info("Step 3: Performing atomic rename...")
|
|
@@ -571,6 +684,50 @@ def _build_restore_command_without_rename(
|
|
|
571
684
|
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
572
685
|
|
|
573
686
|
|
|
687
|
+
def _build_partition_restore_command(
|
|
688
|
+
backup_label: str,
|
|
689
|
+
repo_name: str,
|
|
690
|
+
table: str,
|
|
691
|
+
partitions: list[str],
|
|
692
|
+
database: str,
|
|
693
|
+
backup_timestamp: str,
|
|
694
|
+
rename_suffix: str | None = None,
|
|
695
|
+
) -> str:
|
|
696
|
+
"""Build partition-level restore command with optional AS clause.
|
|
697
|
+
|
|
698
|
+
Args:
|
|
699
|
+
backup_label: Backup snapshot label
|
|
700
|
+
repo_name: Repository name
|
|
701
|
+
table: Table name in format 'database.table'
|
|
702
|
+
partitions: List of partition names to restore
|
|
703
|
+
database: Database name
|
|
704
|
+
backup_timestamp: Backup timestamp
|
|
705
|
+
rename_suffix: Optional suffix for AS clause (e.g., '_restored')
|
|
706
|
+
|
|
707
|
+
Returns:
|
|
708
|
+
SQL RESTORE command string
|
|
709
|
+
"""
|
|
710
|
+
_, table_name = table.split(".", 1)
|
|
711
|
+
|
|
712
|
+
# Build partition list
|
|
713
|
+
partition_list = ", ".join([utils.quote_identifier(p) for p in partitions])
|
|
714
|
+
|
|
715
|
+
# Build table clause
|
|
716
|
+
if rename_suffix:
|
|
717
|
+
# Table only in incremental: use AS clause
|
|
718
|
+
temp_table_name = f"{table_name}{rename_suffix}"
|
|
719
|
+
table_clause = f"TABLE {utils.quote_identifier(table_name)} PARTITION ({partition_list}) AS {utils.quote_identifier(temp_table_name)}"
|
|
720
|
+
else:
|
|
721
|
+
# Table in base: target the _restored table directly (no AS)
|
|
722
|
+
table_clause = f"TABLE {utils.quote_identifier(table_name)} PARTITION ({partition_list})"
|
|
723
|
+
|
|
724
|
+
return f"""RESTORE SNAPSHOT {utils.quote_identifier(backup_label)}
|
|
725
|
+
FROM {utils.quote_identifier(repo_name)}
|
|
726
|
+
DATABASE {utils.quote_identifier(database)}
|
|
727
|
+
ON ({table_clause})
|
|
728
|
+
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
729
|
+
|
|
730
|
+
|
|
574
731
|
def _generate_timestamped_backup_name(table_name: str) -> str:
|
|
575
732
|
"""Generate a timestamped backup table name.
|
|
576
733
|
|
starrocks_br/schema.py
CHANGED
|
@@ -15,71 +15,117 @@
|
|
|
15
15
|
from . import logger
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def initialize_ops_schema(
|
|
18
|
+
def initialize_ops_schema(
|
|
19
|
+
db, ops_database: str = "ops", table_inventory_entries: list[tuple[str, str, str]] | None = None
|
|
20
|
+
) -> None:
|
|
19
21
|
"""Initialize the ops database and all required control tables.
|
|
20
22
|
|
|
21
|
-
Creates empty ops tables.
|
|
22
|
-
|
|
23
|
+
Creates empty ops tables. Optionally populates table_inventory from entries.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
db: Database connection
|
|
27
|
+
ops_database: Name of the ops database (defaults to "ops")
|
|
28
|
+
table_inventory_entries: Optional list of (group, database, table) tuples to bootstrap
|
|
23
29
|
"""
|
|
24
30
|
|
|
25
|
-
logger.info("Creating
|
|
26
|
-
db.execute("CREATE DATABASE IF NOT EXISTS
|
|
27
|
-
logger.success("
|
|
31
|
+
logger.info(f"Creating {ops_database} database...")
|
|
32
|
+
db.execute(f"CREATE DATABASE IF NOT EXISTS {ops_database}")
|
|
33
|
+
logger.success(f"{ops_database} database created")
|
|
34
|
+
|
|
35
|
+
logger.info(f"Creating {ops_database}.table_inventory...")
|
|
36
|
+
db.execute(get_table_inventory_schema(ops_database=ops_database))
|
|
37
|
+
logger.success(f"{ops_database}.table_inventory created")
|
|
38
|
+
|
|
39
|
+
logger.info(f"Creating {ops_database}.backup_history...")
|
|
40
|
+
db.execute(get_backup_history_schema(ops_database=ops_database))
|
|
41
|
+
logger.success(f"{ops_database}.backup_history created")
|
|
28
42
|
|
|
29
|
-
logger.info("Creating
|
|
30
|
-
db.execute(
|
|
31
|
-
logger.success("
|
|
43
|
+
logger.info(f"Creating {ops_database}.restore_history...")
|
|
44
|
+
db.execute(get_restore_history_schema(ops_database=ops_database))
|
|
45
|
+
logger.success(f"{ops_database}.restore_history created")
|
|
32
46
|
|
|
33
|
-
logger.info("Creating
|
|
34
|
-
db.execute(
|
|
35
|
-
logger.success("
|
|
47
|
+
logger.info(f"Creating {ops_database}.run_status...")
|
|
48
|
+
db.execute(get_run_status_schema(ops_database=ops_database))
|
|
49
|
+
logger.success(f"{ops_database}.run_status created")
|
|
36
50
|
|
|
37
|
-
logger.info("Creating
|
|
38
|
-
db.execute(
|
|
39
|
-
logger.success("
|
|
51
|
+
logger.info(f"Creating {ops_database}.backup_partitions...")
|
|
52
|
+
db.execute(get_backup_partitions_schema(ops_database=ops_database))
|
|
53
|
+
logger.success(f"{ops_database}.backup_partitions created")
|
|
40
54
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
55
|
+
if table_inventory_entries:
|
|
56
|
+
logger.info(f"Bootstrapping {ops_database}.table_inventory from configuration...")
|
|
57
|
+
bootstrap_table_inventory(db, table_inventory_entries, ops_database=ops_database)
|
|
58
|
+
logger.success(
|
|
59
|
+
f"{ops_database}.table_inventory bootstrapped with {len(table_inventory_entries)} entries"
|
|
60
|
+
)
|
|
44
61
|
|
|
45
|
-
logger.info("Creating ops.backup_partitions...")
|
|
46
|
-
db.execute(get_backup_partitions_schema())
|
|
47
|
-
logger.success("ops.backup_partitions created")
|
|
48
62
|
logger.info("")
|
|
49
63
|
logger.success("Schema initialized successfully!")
|
|
50
64
|
|
|
51
65
|
|
|
52
|
-
def ensure_ops_schema(db) -> bool:
|
|
66
|
+
def ensure_ops_schema(db, ops_database: str = "ops") -> bool:
|
|
53
67
|
"""Ensure ops schema exists, creating it if necessary.
|
|
54
68
|
|
|
55
69
|
Returns True if schema was created, False if it already existed.
|
|
56
70
|
This is called automatically before backup/restore operations.
|
|
57
71
|
"""
|
|
58
72
|
try:
|
|
59
|
-
result = db.query("SHOW DATABASES LIKE '
|
|
73
|
+
result = db.query(f"SHOW DATABASES LIKE '{ops_database}'")
|
|
60
74
|
|
|
61
75
|
if not result:
|
|
62
|
-
initialize_ops_schema(db)
|
|
76
|
+
initialize_ops_schema(db, ops_database=ops_database)
|
|
63
77
|
return True
|
|
64
78
|
|
|
65
|
-
db.execute("USE
|
|
79
|
+
db.execute(f"USE {ops_database}")
|
|
66
80
|
tables_result = db.query("SHOW TABLES")
|
|
67
81
|
|
|
68
82
|
if not tables_result or len(tables_result) < 5:
|
|
69
|
-
initialize_ops_schema(db)
|
|
83
|
+
initialize_ops_schema(db, ops_database=ops_database)
|
|
70
84
|
return True
|
|
71
85
|
|
|
72
86
|
return False
|
|
73
87
|
|
|
74
88
|
except Exception:
|
|
75
|
-
initialize_ops_schema(db)
|
|
89
|
+
initialize_ops_schema(db, ops_database=ops_database)
|
|
76
90
|
return True
|
|
77
91
|
|
|
78
92
|
|
|
79
|
-
def
|
|
93
|
+
def bootstrap_table_inventory(
|
|
94
|
+
db, entries: list[tuple[str, str, str]], ops_database: str = "ops"
|
|
95
|
+
) -> None:
|
|
96
|
+
"""Bootstrap table_inventory table with entries from configuration.
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
db: Database connection
|
|
100
|
+
entries: List of (group, database, table) tuples
|
|
101
|
+
ops_database: Name of the ops database (defaults to "ops")
|
|
102
|
+
"""
|
|
103
|
+
if not entries:
|
|
104
|
+
return
|
|
105
|
+
|
|
106
|
+
unique_databases = {database for _, database, _ in entries}
|
|
107
|
+
|
|
108
|
+
for database_name in unique_databases:
|
|
109
|
+
result = db.query(f"SHOW DATABASES LIKE '{database_name}'")
|
|
110
|
+
if not result:
|
|
111
|
+
logger.warning(
|
|
112
|
+
f"Database '{database_name}' does not exist. "
|
|
113
|
+
f"Table inventory entries will be created, but backups will fail until the database is created."
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
for group, database, table in entries:
|
|
117
|
+
sql = f"""
|
|
118
|
+
INSERT INTO {ops_database}.table_inventory
|
|
119
|
+
(inventory_group, database_name, table_name)
|
|
120
|
+
VALUES ('{group}', '{database}', '{table}')
|
|
121
|
+
"""
|
|
122
|
+
db.execute(sql)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def get_table_inventory_schema(ops_database: str = "ops") -> str:
|
|
80
126
|
"""Get CREATE TABLE statement for table_inventory."""
|
|
81
|
-
return """
|
|
82
|
-
CREATE TABLE IF NOT EXISTS
|
|
127
|
+
return f"""
|
|
128
|
+
CREATE TABLE IF NOT EXISTS {ops_database}.table_inventory (
|
|
83
129
|
inventory_group STRING NOT NULL COMMENT "Group name for a set of tables",
|
|
84
130
|
database_name STRING NOT NULL COMMENT "Database name",
|
|
85
131
|
table_name STRING NOT NULL COMMENT "Table name, or '*' for all tables in database",
|
|
@@ -92,10 +138,10 @@ def get_table_inventory_schema() -> str:
|
|
|
92
138
|
"""
|
|
93
139
|
|
|
94
140
|
|
|
95
|
-
def get_backup_history_schema() -> str:
|
|
141
|
+
def get_backup_history_schema(ops_database: str = "ops") -> str:
|
|
96
142
|
"""Get CREATE TABLE statement for backup_history."""
|
|
97
|
-
return """
|
|
98
|
-
CREATE TABLE IF NOT EXISTS
|
|
143
|
+
return f"""
|
|
144
|
+
CREATE TABLE IF NOT EXISTS {ops_database}.backup_history (
|
|
99
145
|
label STRING NOT NULL COMMENT "Unique backup snapshot label",
|
|
100
146
|
backup_type STRING NOT NULL COMMENT "Type of backup: full or incremental",
|
|
101
147
|
status STRING NOT NULL COMMENT "Final backup status: FINISHED, FAILED, CANCELLED, TIMEOUT",
|
|
@@ -110,10 +156,10 @@ def get_backup_history_schema() -> str:
|
|
|
110
156
|
"""
|
|
111
157
|
|
|
112
158
|
|
|
113
|
-
def get_restore_history_schema() -> str:
|
|
159
|
+
def get_restore_history_schema(ops_database: str = "ops") -> str:
|
|
114
160
|
"""Get CREATE TABLE statement for restore_history."""
|
|
115
|
-
return """
|
|
116
|
-
CREATE TABLE IF NOT EXISTS
|
|
161
|
+
return f"""
|
|
162
|
+
CREATE TABLE IF NOT EXISTS {ops_database}.restore_history (
|
|
117
163
|
job_id STRING NOT NULL COMMENT "Unique restore job identifier",
|
|
118
164
|
backup_label STRING NOT NULL COMMENT "Source backup snapshot label",
|
|
119
165
|
restore_type STRING NOT NULL COMMENT "Type of restore: partition, table, or database",
|
|
@@ -130,10 +176,10 @@ def get_restore_history_schema() -> str:
|
|
|
130
176
|
"""
|
|
131
177
|
|
|
132
178
|
|
|
133
|
-
def get_run_status_schema() -> str:
|
|
179
|
+
def get_run_status_schema(ops_database: str = "ops") -> str:
|
|
134
180
|
"""Get CREATE TABLE statement for run_status."""
|
|
135
|
-
return """
|
|
136
|
-
CREATE TABLE IF NOT EXISTS
|
|
181
|
+
return f"""
|
|
182
|
+
CREATE TABLE IF NOT EXISTS {ops_database}.run_status (
|
|
137
183
|
scope STRING NOT NULL COMMENT "Job scope: backup or restore",
|
|
138
184
|
label STRING NOT NULL COMMENT "Job label or identifier",
|
|
139
185
|
state STRING NOT NULL DEFAULT "ACTIVE" COMMENT "Job state: ACTIVE, FINISHED, FAILED, or CANCELLED",
|
|
@@ -145,12 +191,12 @@ def get_run_status_schema() -> str:
|
|
|
145
191
|
"""
|
|
146
192
|
|
|
147
193
|
|
|
148
|
-
def get_backup_partitions_schema() -> str:
|
|
194
|
+
def get_backup_partitions_schema(ops_database: str = "ops") -> str:
|
|
149
195
|
"""Get CREATE TABLE statement for backup_partitions."""
|
|
150
|
-
return """
|
|
151
|
-
CREATE TABLE IF NOT EXISTS
|
|
196
|
+
return f"""
|
|
197
|
+
CREATE TABLE IF NOT EXISTS {ops_database}.backup_partitions (
|
|
152
198
|
key_hash STRING NOT NULL COMMENT "MD5 hash of composite key (label, database_name, table_name, partition_name)",
|
|
153
|
-
label STRING NOT NULL COMMENT "The backup label this partition belongs to. FK to
|
|
199
|
+
label STRING NOT NULL COMMENT "The backup label this partition belongs to. FK to {ops_database}.backup_history.label.",
|
|
154
200
|
database_name STRING NOT NULL COMMENT "The name of the database the partition belongs to.",
|
|
155
201
|
table_name STRING NOT NULL COMMENT "The name of the table the partition belongs to.",
|
|
156
202
|
partition_name STRING NOT NULL COMMENT "The name of the specific partition.",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: starrocks-br
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: StarRocks Backup and Restore automation tool
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -95,6 +95,15 @@ port: 9030 # MySQL protocol port
|
|
|
95
95
|
user: "root" # Database user with backup/restore privileges
|
|
96
96
|
database: "your_database" # Database containing tables to backup
|
|
97
97
|
repository: "your_repo_name" # Repository created via CREATE REPOSITORY in StarRocks
|
|
98
|
+
|
|
99
|
+
# Optional: Define table inventory groups directly in config
|
|
100
|
+
table_inventory:
|
|
101
|
+
- group: "production"
|
|
102
|
+
tables:
|
|
103
|
+
- database: "mydb"
|
|
104
|
+
table: "users"
|
|
105
|
+
- database: "mydb"
|
|
106
|
+
table: "orders"
|
|
98
107
|
```
|
|
99
108
|
|
|
100
109
|
Set password:
|
|
@@ -111,7 +120,11 @@ See [Configuration Reference](docs/configuration.md) for TLS and advanced option
|
|
|
111
120
|
starrocks-br init --config config.yaml
|
|
112
121
|
```
|
|
113
122
|
|
|
114
|
-
|
|
123
|
+
This creates the `ops` database and automatically populates table inventory from your config (if defined).
|
|
124
|
+
|
|
125
|
+
**Note:** If you modify the `table_inventory` in your config file, rerun `starrocks-br init --config config.yaml` to update the database.
|
|
126
|
+
|
|
127
|
+
**Alternative: Define inventory groups manually** (in StarRocks):
|
|
115
128
|
```sql
|
|
116
129
|
INSERT INTO ops.table_inventory (inventory_group, database_name, table_name)
|
|
117
130
|
VALUES
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
starrocks_br/__init__.py,sha256=SeQqVuym4h6mvf4ar0UpBs_qDWbP0XOYH65Qj7WT4HI,600
|
|
2
|
+
starrocks_br/cli.py,sha256=xvsL_lv3sjVFCpASfsEi_jRztK4KqnjOt1ee0pTXTTA,25996
|
|
3
|
+
starrocks_br/concurrency.py,sha256=k06-9hPdAayXGnN_9gfJNkdZTVGnFsHGsws5RK7GauE,6586
|
|
4
|
+
starrocks_br/config.py,sha256=JooRqqNm39UqbXSFqgFn3x84heOb-i0ZezNKpRTxqj0,5463
|
|
5
|
+
starrocks_br/db.py,sha256=abGG12e4JN4ydrz1ZdahKvOkk9CKbkGbkLnRxjUMFA8,5518
|
|
6
|
+
starrocks_br/error_handler.py,sha256=PnmjUWDzP5P_5SymT-cYTiyrSe8M0DDW4pXUNYv_HOc,15890
|
|
7
|
+
starrocks_br/exceptions.py,sha256=i1XUEmV1q60oTSN8u3xb7SQb3ZCaCcKvvAua4ZXd0lM,5061
|
|
8
|
+
starrocks_br/executor.py,sha256=w3jMKuchdqlNf5xDA6L-AagQ1yHgktzhJyARBu6xCp0,12238
|
|
9
|
+
starrocks_br/health.py,sha256=4CEX6nFKN1Vdc2cP_b89IVFZq6eaHKGMrs-FFDC4T9Q,1594
|
|
10
|
+
starrocks_br/history.py,sha256=UOQqw8-bKfImQkvq6_gIIb8i7R5BaIjKiDiku-koaas,3621
|
|
11
|
+
starrocks_br/labels.py,sha256=IqoNgPd-LCbZYGrVZxa0Lj9TNa8XcNCxp1oU2yiLHHU,2397
|
|
12
|
+
starrocks_br/logger.py,sha256=a44M_VDhVqnaU5sxqJ0tVoXBdY4tAmWG5P3DRNOdbWE,2004
|
|
13
|
+
starrocks_br/planner.py,sha256=B0K9Qi0ILoEJq_oc6q8sPBofALLSBpo94QgbjzCl7MU,12758
|
|
14
|
+
starrocks_br/repository.py,sha256=hcl5ehoHdi0wgWKYl1wQYVjSpkdgrpSUPbDS2vV6BwM,1807
|
|
15
|
+
starrocks_br/restore.py,sha256=mLU42XbeSr6zcLOCrbiZA4JXpwBBHITJCD4DpZcFS9w,26654
|
|
16
|
+
starrocks_br/schema.py,sha256=KCQEm6h6umEWA1lb80_Sfa8ZDBARMcWNZYcF2UAlMLs,9072
|
|
17
|
+
starrocks_br/timezone.py,sha256=DLoOJ2-Z65a__PEcZS1eYauVIbRJNWOOX5lpfdHfqGc,4150
|
|
18
|
+
starrocks_br/utils.py,sha256=bmEZpD_CQ1_7hy51ZkRG7KKYOglhqW3EZYsWwFPUPow,2749
|
|
19
|
+
starrocks_br-0.6.0.dist-info/licenses/LICENSE,sha256=HV8aogcPFdRayUZanMcDj73f-B3qg2YstudemA_Qi5U,11369
|
|
20
|
+
starrocks_br-0.6.0.dist-info/METADATA,sha256=A_Tpu-zjUqB97DyD2LvmzRDc3Yyvj4SDq1VmJOzDGWs,6412
|
|
21
|
+
starrocks_br-0.6.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
22
|
+
starrocks_br-0.6.0.dist-info/entry_points.txt,sha256=AKUt01G2MAlh85s1Q9kNQDOUio14kaTnT3dmg9gjdNg,54
|
|
23
|
+
starrocks_br-0.6.0.dist-info/top_level.txt,sha256=CU1tGVo0kjulhDr761Sndg-oTeRKsisDnWm8UG95aBE,13
|
|
24
|
+
starrocks_br-0.6.0.dist-info/RECORD,,
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
starrocks_br/__init__.py,sha256=SeQqVuym4h6mvf4ar0UpBs_qDWbP0XOYH65Qj7WT4HI,600
|
|
2
|
-
starrocks_br/cli.py,sha256=9DQzFDYwqDVfoQSNamHBtQrwBKH99agsIFWrCs096wM,23496
|
|
3
|
-
starrocks_br/concurrency.py,sha256=aLHLjXy8ccmtfOXUuL-HJx_xQYIR0lpQE4TI4LI9y1k,6130
|
|
4
|
-
starrocks_br/config.py,sha256=6A4BdHC7OT6RQSpK_cZ_8e8M6oHIMJwu6jKDa4ZtHRg,2822
|
|
5
|
-
starrocks_br/db.py,sha256=abGG12e4JN4ydrz1ZdahKvOkk9CKbkGbkLnRxjUMFA8,5518
|
|
6
|
-
starrocks_br/error_handler.py,sha256=vHNaEiLkTVBkhcrNJyjboaqikpxeTMA0ZsNsE01ysB8,13657
|
|
7
|
-
starrocks_br/exceptions.py,sha256=Z9zoiCdSjKAxYKIKf0rwYjTlkoCichIz7WwoFIfnmi8,4482
|
|
8
|
-
starrocks_br/executor.py,sha256=aQA6lQuUc0TGy9CqqpAi9f7JDJsdwatmI-TvETuFTPs,12016
|
|
9
|
-
starrocks_br/health.py,sha256=4CEX6nFKN1Vdc2cP_b89IVFZq6eaHKGMrs-FFDC4T9Q,1594
|
|
10
|
-
starrocks_br/history.py,sha256=FM8uwqjIuDbKYJSf1XKr1Sila98a6uz1CESD2MsVjtI,3536
|
|
11
|
-
starrocks_br/labels.py,sha256=93AROzHxTa5eCEVZlqJi9Jjv7w-YpYTANqTu-n3ujP8,2234
|
|
12
|
-
starrocks_br/logger.py,sha256=a44M_VDhVqnaU5sxqJ0tVoXBdY4tAmWG5P3DRNOdbWE,2004
|
|
13
|
-
starrocks_br/planner.py,sha256=zJwOx16y3Chn_WEmPgKdJ_Zfy-8Hfnp_w212qdrfxao,11326
|
|
14
|
-
starrocks_br/repository.py,sha256=jYzu2EMR7RPp86c6PkuHTA-cld6qX-Jj0yTD-lryZR0,1823
|
|
15
|
-
starrocks_br/restore.py,sha256=B9m6bFONyUiUGTYnb6ac3CUpW-Ev-S820-q-LD7R5Po,20677
|
|
16
|
-
starrocks_br/schema.py,sha256=KayDhOL3NQq_ShwCvGrTxbFkvX7BaBybfEAe2Dxuuu0,6771
|
|
17
|
-
starrocks_br/timezone.py,sha256=DLoOJ2-Z65a__PEcZS1eYauVIbRJNWOOX5lpfdHfqGc,4150
|
|
18
|
-
starrocks_br/utils.py,sha256=bmEZpD_CQ1_7hy51ZkRG7KKYOglhqW3EZYsWwFPUPow,2749
|
|
19
|
-
starrocks_br-0.5.2.dist-info/licenses/LICENSE,sha256=HV8aogcPFdRayUZanMcDj73f-B3qg2YstudemA_Qi5U,11369
|
|
20
|
-
starrocks_br-0.5.2.dist-info/METADATA,sha256=Np4XYVJQmW3hkIkpbfCeOvTZmFHLLYLzUqUV9ShHU6A,5931
|
|
21
|
-
starrocks_br-0.5.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
22
|
-
starrocks_br-0.5.2.dist-info/entry_points.txt,sha256=AKUt01G2MAlh85s1Q9kNQDOUio14kaTnT3dmg9gjdNg,54
|
|
23
|
-
starrocks_br-0.5.2.dist-info/top_level.txt,sha256=CU1tGVo0kjulhDr761Sndg-oTeRKsisDnWm8UG95aBE,13
|
|
24
|
-
starrocks_br-0.5.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|