starrocks-br 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- starrocks_br/__init__.py +1 -0
- starrocks_br/cli.py +385 -0
- starrocks_br/concurrency.py +177 -0
- starrocks_br/config.py +41 -0
- starrocks_br/db.py +88 -0
- starrocks_br/executor.py +245 -0
- starrocks_br/health.py +34 -0
- starrocks_br/history.py +93 -0
- starrocks_br/labels.py +52 -0
- starrocks_br/logger.py +36 -0
- starrocks_br/planner.py +280 -0
- starrocks_br/repository.py +36 -0
- starrocks_br/restore.py +493 -0
- starrocks_br/schema.py +144 -0
- starrocks_br-0.1.0.dist-info/METADATA +12 -0
- starrocks_br-0.1.0.dist-info/RECORD +19 -0
- starrocks_br-0.1.0.dist-info/WHEEL +5 -0
- starrocks_br-0.1.0.dist-info/entry_points.txt +2 -0
- starrocks_br-0.1.0.dist-info/top_level.txt +1 -0
starrocks_br/restore.py
ADDED
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import datetime
|
|
3
|
+
from typing import Dict, List, Optional
|
|
4
|
+
from . import history, concurrency, logger
|
|
5
|
+
|
|
6
|
+
MAX_POLLS = 21600 # 6 hours
|
|
7
|
+
|
|
8
|
+
def get_snapshot_timestamp(db, repo_name: str, snapshot_name: str) -> str:
|
|
9
|
+
"""Get the backup timestamp for a specific snapshot from the repository.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
db: Database connection
|
|
13
|
+
repo_name: Repository name
|
|
14
|
+
snapshot_name: Snapshot name to look up
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
The backup timestamp string
|
|
18
|
+
|
|
19
|
+
Raises:
|
|
20
|
+
ValueError: If snapshot is not found in the repository
|
|
21
|
+
"""
|
|
22
|
+
query = f"SHOW SNAPSHOT ON {repo_name} WHERE Snapshot = '{snapshot_name}'"
|
|
23
|
+
|
|
24
|
+
rows = db.query(query)
|
|
25
|
+
if not rows:
|
|
26
|
+
raise ValueError(f"Snapshot '{snapshot_name}' not found in repository '{repo_name}'")
|
|
27
|
+
|
|
28
|
+
# The result should be a single row with columns: Snapshot, Timestamp, Status
|
|
29
|
+
result = rows[0]
|
|
30
|
+
|
|
31
|
+
if isinstance(result, dict):
|
|
32
|
+
timestamp = result.get("Timestamp")
|
|
33
|
+
else:
|
|
34
|
+
timestamp = result[1] if len(result) > 1 else None
|
|
35
|
+
|
|
36
|
+
if not timestamp:
|
|
37
|
+
raise ValueError(f"Could not extract timestamp for snapshot '{snapshot_name}'")
|
|
38
|
+
|
|
39
|
+
return timestamp
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def build_partition_restore_command(
|
|
43
|
+
database: str,
|
|
44
|
+
table: str,
|
|
45
|
+
partition: str,
|
|
46
|
+
backup_label: str,
|
|
47
|
+
repository: str,
|
|
48
|
+
backup_timestamp: str,
|
|
49
|
+
) -> str:
|
|
50
|
+
"""Build RESTORE command for single partition recovery."""
|
|
51
|
+
return f"""RESTORE SNAPSHOT {backup_label}
|
|
52
|
+
FROM {repository}
|
|
53
|
+
DATABASE {database}
|
|
54
|
+
ON (TABLE {table} PARTITION ({partition}))
|
|
55
|
+
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def build_table_restore_command(
|
|
59
|
+
database: str,
|
|
60
|
+
table: str,
|
|
61
|
+
backup_label: str,
|
|
62
|
+
repository: str,
|
|
63
|
+
backup_timestamp: str,
|
|
64
|
+
) -> str:
|
|
65
|
+
"""Build RESTORE command for full table recovery."""
|
|
66
|
+
return f"""RESTORE SNAPSHOT {backup_label}
|
|
67
|
+
FROM {repository}
|
|
68
|
+
DATABASE {database}
|
|
69
|
+
ON (TABLE {table})
|
|
70
|
+
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def build_database_restore_command(
|
|
74
|
+
database: str,
|
|
75
|
+
backup_label: str,
|
|
76
|
+
repository: str,
|
|
77
|
+
backup_timestamp: str,
|
|
78
|
+
) -> str:
|
|
79
|
+
"""Build RESTORE command for full database recovery."""
|
|
80
|
+
return f"""RESTORE SNAPSHOT {backup_label}
|
|
81
|
+
FROM {repository}
|
|
82
|
+
DATABASE {database}
|
|
83
|
+
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def poll_restore_status(db, label: str, database: str, max_polls: int = MAX_POLLS, poll_interval: float = 1.0) -> Dict[str, str]:
|
|
87
|
+
"""Poll restore status until completion or timeout.
|
|
88
|
+
|
|
89
|
+
Note: SHOW RESTORE only returns the LAST restore in a database.
|
|
90
|
+
We verify that the Label matches our expected label.
|
|
91
|
+
|
|
92
|
+
Important: If we see a different label, it means another restore
|
|
93
|
+
operation overwrote ours and we've lost tracking (race condition).
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
db: Database connection
|
|
97
|
+
label: Expected snapshot label to monitor
|
|
98
|
+
database: Database name where restore was submitted
|
|
99
|
+
max_polls: Maximum number of polling attempts
|
|
100
|
+
poll_interval: Seconds to wait between polls
|
|
101
|
+
|
|
102
|
+
Returns dictionary with keys: state, label
|
|
103
|
+
Possible states: FINISHED, CANCELLED, TIMEOUT, ERROR, LOST
|
|
104
|
+
"""
|
|
105
|
+
query = f"SHOW RESTORE FROM {database}"
|
|
106
|
+
first_poll = True
|
|
107
|
+
last_state = None
|
|
108
|
+
poll_count = 0
|
|
109
|
+
|
|
110
|
+
for _ in range(max_polls):
|
|
111
|
+
poll_count += 1
|
|
112
|
+
try:
|
|
113
|
+
rows = db.query(query)
|
|
114
|
+
|
|
115
|
+
if not rows:
|
|
116
|
+
time.sleep(poll_interval)
|
|
117
|
+
continue
|
|
118
|
+
|
|
119
|
+
result = rows[0]
|
|
120
|
+
|
|
121
|
+
if isinstance(result, dict):
|
|
122
|
+
snapshot_label = result.get("Label", "")
|
|
123
|
+
state = result.get("State", "UNKNOWN")
|
|
124
|
+
else:
|
|
125
|
+
# Tuple format: JobId, Label, Timestamp, DbName, State, ...
|
|
126
|
+
snapshot_label = result[1] if len(result) > 1 else ""
|
|
127
|
+
state = result[4] if len(result) > 4 else "UNKNOWN"
|
|
128
|
+
|
|
129
|
+
if snapshot_label != label and snapshot_label:
|
|
130
|
+
if first_poll:
|
|
131
|
+
first_poll = False
|
|
132
|
+
time.sleep(poll_interval)
|
|
133
|
+
continue
|
|
134
|
+
else:
|
|
135
|
+
return {"state": "LOST", "label": label}
|
|
136
|
+
|
|
137
|
+
first_poll = False
|
|
138
|
+
|
|
139
|
+
if state != last_state or poll_count % 10 == 0:
|
|
140
|
+
logger.progress(f"Restore status: {state} (poll {poll_count}/{max_polls})")
|
|
141
|
+
last_state = state
|
|
142
|
+
|
|
143
|
+
if state in ["FINISHED", "CANCELLED", "UNKNOWN"]:
|
|
144
|
+
return {"state": state, "label": label}
|
|
145
|
+
|
|
146
|
+
time.sleep(poll_interval)
|
|
147
|
+
|
|
148
|
+
except Exception:
|
|
149
|
+
return {"state": "ERROR", "label": label}
|
|
150
|
+
|
|
151
|
+
return {"state": "TIMEOUT", "label": label}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def execute_restore(
|
|
155
|
+
db,
|
|
156
|
+
restore_command: str,
|
|
157
|
+
backup_label: str,
|
|
158
|
+
restore_type: str,
|
|
159
|
+
repository: str,
|
|
160
|
+
database: str,
|
|
161
|
+
max_polls: int = MAX_POLLS,
|
|
162
|
+
poll_interval: float = 1.0,
|
|
163
|
+
scope: str = "restore",
|
|
164
|
+
) -> Dict:
|
|
165
|
+
"""Execute a complete restore workflow: submit command and monitor progress.
|
|
166
|
+
|
|
167
|
+
Returns dictionary with keys: success, final_status, error_message
|
|
168
|
+
"""
|
|
169
|
+
started_at = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
170
|
+
|
|
171
|
+
try:
|
|
172
|
+
db.execute(restore_command.strip())
|
|
173
|
+
except Exception as e:
|
|
174
|
+
logger.error(f"Failed to submit restore command: {str(e)}")
|
|
175
|
+
return {
|
|
176
|
+
"success": False,
|
|
177
|
+
"final_status": None,
|
|
178
|
+
"error_message": f"Failed to submit restore command: {str(e)}"
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
label = backup_label
|
|
182
|
+
|
|
183
|
+
try:
|
|
184
|
+
final_status = poll_restore_status(db, label, database, max_polls, poll_interval)
|
|
185
|
+
|
|
186
|
+
success = final_status["state"] == "FINISHED"
|
|
187
|
+
finished_at = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
188
|
+
|
|
189
|
+
try:
|
|
190
|
+
history.log_restore(
|
|
191
|
+
db,
|
|
192
|
+
{
|
|
193
|
+
"job_id": label,
|
|
194
|
+
"backup_label": backup_label,
|
|
195
|
+
"restore_type": restore_type,
|
|
196
|
+
"status": final_status["state"],
|
|
197
|
+
"repository": repository,
|
|
198
|
+
"started_at": started_at,
|
|
199
|
+
"finished_at": finished_at,
|
|
200
|
+
"error_message": None if success else final_status["state"],
|
|
201
|
+
},
|
|
202
|
+
)
|
|
203
|
+
except Exception as e:
|
|
204
|
+
logger.error(f"Failed to log restore history: {str(e)}")
|
|
205
|
+
|
|
206
|
+
try:
|
|
207
|
+
concurrency.complete_job_slot(db, scope=scope, label=label, final_state=final_status["state"])
|
|
208
|
+
except Exception as e:
|
|
209
|
+
logger.error(f"Failed to complete job slot: {str(e)}")
|
|
210
|
+
|
|
211
|
+
return {
|
|
212
|
+
"success": success,
|
|
213
|
+
"final_status": final_status,
|
|
214
|
+
"error_message": None if success else f"Restore failed with state: {final_status['state']}"
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
except Exception as e:
|
|
218
|
+
logger.error(f"Restore execution failed: {str(e)}")
|
|
219
|
+
return {
|
|
220
|
+
"success": False,
|
|
221
|
+
"final_status": None,
|
|
222
|
+
"error_message": str(e)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def find_restore_pair(db, target_label: str) -> List[str]:
|
|
227
|
+
"""Find the correct sequence of backups needed for restore.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
db: Database connection
|
|
231
|
+
target_label: The backup label to restore to
|
|
232
|
+
|
|
233
|
+
Returns:
|
|
234
|
+
List of backup labels in restore order [base_full_backup, target_label]
|
|
235
|
+
or [target_label] if target is a full backup
|
|
236
|
+
|
|
237
|
+
Raises:
|
|
238
|
+
ValueError: If target label not found or incremental has no preceding full backup
|
|
239
|
+
"""
|
|
240
|
+
query = f"""
|
|
241
|
+
SELECT label, backup_type, finished_at
|
|
242
|
+
FROM ops.backup_history
|
|
243
|
+
WHERE label = '{target_label}'
|
|
244
|
+
AND status = 'FINISHED'
|
|
245
|
+
"""
|
|
246
|
+
|
|
247
|
+
rows = db.query(query)
|
|
248
|
+
if not rows:
|
|
249
|
+
raise ValueError(f"Backup label '{target_label}' not found or not successful")
|
|
250
|
+
|
|
251
|
+
target_info = {
|
|
252
|
+
"label": rows[0][0],
|
|
253
|
+
"backup_type": rows[0][1],
|
|
254
|
+
"finished_at": rows[0][2]
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
if target_info["backup_type"] == "full":
|
|
258
|
+
return [target_label]
|
|
259
|
+
|
|
260
|
+
if target_info["backup_type"] == "incremental":
|
|
261
|
+
database_name = target_label.split('_')[0]
|
|
262
|
+
|
|
263
|
+
full_backup_query = f"""
|
|
264
|
+
SELECT label, backup_type, finished_at
|
|
265
|
+
FROM ops.backup_history
|
|
266
|
+
WHERE backup_type = 'full'
|
|
267
|
+
AND status = 'FINISHED'
|
|
268
|
+
AND label LIKE '{database_name}_%'
|
|
269
|
+
AND finished_at < '{target_info["finished_at"]}'
|
|
270
|
+
ORDER BY finished_at DESC
|
|
271
|
+
LIMIT 1
|
|
272
|
+
"""
|
|
273
|
+
|
|
274
|
+
full_rows = db.query(full_backup_query)
|
|
275
|
+
if not full_rows:
|
|
276
|
+
raise ValueError(f"No successful full backup found before incremental '{target_label}'")
|
|
277
|
+
|
|
278
|
+
base_full_backup = full_rows[0][0]
|
|
279
|
+
return [base_full_backup, target_label]
|
|
280
|
+
|
|
281
|
+
raise ValueError(f"Unknown backup type '{target_info['backup_type']}' for label '{target_label}'")
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def get_tables_from_backup(db, label: str, group: Optional[str] = None) -> List[str]:
|
|
285
|
+
"""Get list of tables to restore from backup manifest.
|
|
286
|
+
|
|
287
|
+
Args:
|
|
288
|
+
db: Database connection
|
|
289
|
+
label: Backup label
|
|
290
|
+
group: Optional inventory group to filter tables
|
|
291
|
+
|
|
292
|
+
Returns:
|
|
293
|
+
List of table names to restore
|
|
294
|
+
"""
|
|
295
|
+
query = f"""
|
|
296
|
+
SELECT DISTINCT database_name, table_name
|
|
297
|
+
FROM ops.backup_partitions
|
|
298
|
+
WHERE label = '{label}'
|
|
299
|
+
ORDER BY database_name, table_name
|
|
300
|
+
"""
|
|
301
|
+
|
|
302
|
+
rows = db.query(query)
|
|
303
|
+
if not rows:
|
|
304
|
+
return []
|
|
305
|
+
|
|
306
|
+
tables = [f"{row[0]}.{row[1]}" for row in rows]
|
|
307
|
+
|
|
308
|
+
if group:
|
|
309
|
+
group_query = f"""
|
|
310
|
+
SELECT database_name, table_name
|
|
311
|
+
FROM ops.table_inventory
|
|
312
|
+
WHERE inventory_group = '{group}'
|
|
313
|
+
"""
|
|
314
|
+
|
|
315
|
+
group_rows = db.query(group_query)
|
|
316
|
+
if not group_rows:
|
|
317
|
+
return []
|
|
318
|
+
|
|
319
|
+
group_tables = {f"{row[0]}.{row[1]}" for row in group_rows}
|
|
320
|
+
|
|
321
|
+
tables = [table for table in tables if table in group_tables]
|
|
322
|
+
|
|
323
|
+
return tables
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def execute_restore_flow(db, repo_name: str, restore_pair: List[str], tables_to_restore: List[str], rename_suffix: str = "_restored") -> Dict:
|
|
327
|
+
"""Execute the complete restore flow with safety measures.
|
|
328
|
+
|
|
329
|
+
Args:
|
|
330
|
+
db: Database connection
|
|
331
|
+
repo_name: Repository name
|
|
332
|
+
restore_pair: List of backup labels in restore order
|
|
333
|
+
tables_to_restore: List of tables to restore (format: database.table)
|
|
334
|
+
rename_suffix: Suffix for temporary tables
|
|
335
|
+
|
|
336
|
+
Returns:
|
|
337
|
+
Dictionary with success status and details
|
|
338
|
+
"""
|
|
339
|
+
if not restore_pair:
|
|
340
|
+
return {
|
|
341
|
+
"success": False,
|
|
342
|
+
"error_message": "No restore pair provided"
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
if not tables_to_restore:
|
|
346
|
+
return {
|
|
347
|
+
"success": False,
|
|
348
|
+
"error_message": "No tables to restore"
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
logger.info("")
|
|
352
|
+
logger.info("=== RESTORE PLAN ===")
|
|
353
|
+
logger.info(f"Repository: {repo_name}")
|
|
354
|
+
logger.info(f"Restore sequence: {' -> '.join(restore_pair)}")
|
|
355
|
+
logger.info(f"Tables to restore: {', '.join(tables_to_restore)}")
|
|
356
|
+
logger.info(f"Temporary table suffix: {rename_suffix}")
|
|
357
|
+
logger.info("")
|
|
358
|
+
logger.info("This will restore data to temporary tables and then perform atomic rename.")
|
|
359
|
+
logger.warning("WARNING: This operation will replace existing tables!")
|
|
360
|
+
|
|
361
|
+
confirmation = input("\nDo you want to proceed? [Y/n]: ").strip()
|
|
362
|
+
if confirmation.lower() != 'y':
|
|
363
|
+
return {
|
|
364
|
+
"success": False,
|
|
365
|
+
"error_message": "Restore operation cancelled by user"
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
database_name = tables_to_restore[0].split('.')[0]
|
|
370
|
+
|
|
371
|
+
base_label = restore_pair[0]
|
|
372
|
+
logger.info("")
|
|
373
|
+
logger.info(f"Step 1: Restoring base backup '{base_label}'...")
|
|
374
|
+
|
|
375
|
+
base_timestamp = get_snapshot_timestamp(db, repo_name, base_label)
|
|
376
|
+
|
|
377
|
+
base_restore_command = _build_restore_command_with_rename(
|
|
378
|
+
base_label, repo_name, tables_to_restore, rename_suffix, database_name, base_timestamp
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
base_result = execute_restore(
|
|
382
|
+
db, base_restore_command, base_label, "full", repo_name, database_name, scope="restore"
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
if not base_result["success"]:
|
|
386
|
+
return {
|
|
387
|
+
"success": False,
|
|
388
|
+
"error_message": f"Base restore failed: {base_result['error_message']}"
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
logger.success("Base restore completed successfully")
|
|
392
|
+
|
|
393
|
+
if len(restore_pair) > 1:
|
|
394
|
+
incremental_label = restore_pair[1]
|
|
395
|
+
logger.info("")
|
|
396
|
+
logger.info(f"Step 2: Applying incremental backup '{incremental_label}'...")
|
|
397
|
+
|
|
398
|
+
incremental_timestamp = get_snapshot_timestamp(db, repo_name, incremental_label)
|
|
399
|
+
|
|
400
|
+
incremental_restore_command = _build_restore_command_without_rename(
|
|
401
|
+
incremental_label, repo_name, tables_to_restore, database_name, incremental_timestamp
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
incremental_result = execute_restore(
|
|
405
|
+
db, incremental_restore_command, incremental_label, "incremental", repo_name, database_name, scope="restore"
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
if not incremental_result["success"]:
|
|
409
|
+
return {
|
|
410
|
+
"success": False,
|
|
411
|
+
"error_message": f"Incremental restore failed: {incremental_result['error_message']}"
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
logger.success("Incremental restore completed successfully")
|
|
415
|
+
|
|
416
|
+
logger.info("")
|
|
417
|
+
logger.info("Step 3: Performing atomic rename...")
|
|
418
|
+
rename_result = _perform_atomic_rename(db, tables_to_restore, rename_suffix)
|
|
419
|
+
|
|
420
|
+
if not rename_result["success"]:
|
|
421
|
+
return {
|
|
422
|
+
"success": False,
|
|
423
|
+
"error_message": f"Atomic rename failed: {rename_result['error_message']}"
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
logger.success("Atomic rename completed successfully")
|
|
427
|
+
|
|
428
|
+
return {
|
|
429
|
+
"success": True,
|
|
430
|
+
"message": f"Restore completed successfully. Restored {len(tables_to_restore)} tables."
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
except Exception as e:
|
|
434
|
+
return {
|
|
435
|
+
"success": False,
|
|
436
|
+
"error_message": f"Restore flow failed: {str(e)}"
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _build_restore_command_with_rename(backup_label: str, repo_name: str, tables: List[str], rename_suffix: str, database: str, backup_timestamp: str) -> str:
|
|
441
|
+
"""Build restore command with AS clause for temporary table names."""
|
|
442
|
+
table_clauses = []
|
|
443
|
+
for table in tables:
|
|
444
|
+
_, table_name = table.split('.', 1)
|
|
445
|
+
temp_table_name = f"{table_name}{rename_suffix}"
|
|
446
|
+
table_clauses.append(f"TABLE {table_name} AS {temp_table_name}")
|
|
447
|
+
|
|
448
|
+
on_clause = ",\n ".join(table_clauses)
|
|
449
|
+
|
|
450
|
+
return f"""RESTORE SNAPSHOT {backup_label}
|
|
451
|
+
FROM {repo_name}
|
|
452
|
+
DATABASE {database}
|
|
453
|
+
ON ({on_clause})
|
|
454
|
+
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def _build_restore_command_without_rename(backup_label: str, repo_name: str, tables: List[str], database: str, backup_timestamp: str) -> str:
|
|
458
|
+
"""Build restore command without AS clause (for incremental restores to existing temp tables)."""
|
|
459
|
+
table_clauses = []
|
|
460
|
+
for table in tables:
|
|
461
|
+
_, table_name = table.split('.', 1)
|
|
462
|
+
table_clauses.append(f"TABLE {table_name}")
|
|
463
|
+
|
|
464
|
+
on_clause = ",\n ".join(table_clauses)
|
|
465
|
+
|
|
466
|
+
return f"""RESTORE SNAPSHOT {backup_label}
|
|
467
|
+
FROM {repo_name}
|
|
468
|
+
DATABASE {database}
|
|
469
|
+
ON ({on_clause})
|
|
470
|
+
PROPERTIES ("backup_timestamp" = "{backup_timestamp}")"""
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _perform_atomic_rename(db, tables: List[str], rename_suffix: str) -> Dict:
|
|
474
|
+
"""Perform atomic rename of temporary tables to make them live."""
|
|
475
|
+
try:
|
|
476
|
+
rename_statements = []
|
|
477
|
+
for table in tables:
|
|
478
|
+
database, table_name = table.split('.', 1)
|
|
479
|
+
temp_table_name = f"{table_name}{rename_suffix}"
|
|
480
|
+
|
|
481
|
+
rename_statements.append(f"ALTER TABLE {database}.{table_name} RENAME {table_name}_backup")
|
|
482
|
+
rename_statements.append(f"ALTER TABLE {database}.{temp_table_name} RENAME {table_name}")
|
|
483
|
+
|
|
484
|
+
for statement in rename_statements:
|
|
485
|
+
db.execute(statement)
|
|
486
|
+
|
|
487
|
+
return {"success": True}
|
|
488
|
+
|
|
489
|
+
except Exception as e:
|
|
490
|
+
return {
|
|
491
|
+
"success": False,
|
|
492
|
+
"error_message": f"Failed to perform atomic rename: {str(e)}"
|
|
493
|
+
}
|
starrocks_br/schema.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
from . import logger
|
|
2
|
+
|
|
3
|
+
def initialize_ops_schema(db) -> None:
|
|
4
|
+
"""Initialize the ops database and all required control tables.
|
|
5
|
+
|
|
6
|
+
Creates empty ops tables. Does NOT populate with sample data.
|
|
7
|
+
Users must manually insert their table inventory records.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
logger.info("Creating ops database...")
|
|
11
|
+
db.execute("CREATE DATABASE IF NOT EXISTS ops")
|
|
12
|
+
logger.success("ops database created")
|
|
13
|
+
|
|
14
|
+
logger.info("Creating ops.table_inventory...")
|
|
15
|
+
db.execute(get_table_inventory_schema())
|
|
16
|
+
logger.success("ops.table_inventory created")
|
|
17
|
+
|
|
18
|
+
logger.info("Creating ops.backup_history...")
|
|
19
|
+
db.execute(get_backup_history_schema())
|
|
20
|
+
logger.success("ops.backup_history created")
|
|
21
|
+
|
|
22
|
+
logger.info("Creating ops.restore_history...")
|
|
23
|
+
db.execute(get_restore_history_schema())
|
|
24
|
+
logger.success("ops.restore_history created")
|
|
25
|
+
|
|
26
|
+
logger.info("Creating ops.run_status...")
|
|
27
|
+
db.execute(get_run_status_schema())
|
|
28
|
+
logger.success("ops.run_status created")
|
|
29
|
+
|
|
30
|
+
logger.info("Creating ops.backup_partitions...")
|
|
31
|
+
db.execute(get_backup_partitions_schema())
|
|
32
|
+
logger.success("ops.backup_partitions created")
|
|
33
|
+
logger.info("")
|
|
34
|
+
logger.success("Schema initialized successfully!")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def ensure_ops_schema(db) -> bool:
|
|
38
|
+
"""Ensure ops schema exists, creating it if necessary.
|
|
39
|
+
|
|
40
|
+
Returns True if schema was created, False if it already existed.
|
|
41
|
+
This is called automatically before backup/restore operations.
|
|
42
|
+
"""
|
|
43
|
+
try:
|
|
44
|
+
result = db.query("SHOW DATABASES LIKE 'ops'")
|
|
45
|
+
|
|
46
|
+
if not result:
|
|
47
|
+
initialize_ops_schema(db)
|
|
48
|
+
return True
|
|
49
|
+
|
|
50
|
+
db.execute("USE ops")
|
|
51
|
+
tables_result = db.query("SHOW TABLES")
|
|
52
|
+
|
|
53
|
+
if not tables_result or len(tables_result) < 5:
|
|
54
|
+
initialize_ops_schema(db)
|
|
55
|
+
return True
|
|
56
|
+
|
|
57
|
+
return False
|
|
58
|
+
|
|
59
|
+
except Exception:
|
|
60
|
+
initialize_ops_schema(db)
|
|
61
|
+
return True
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def get_table_inventory_schema() -> str:
|
|
65
|
+
"""Get CREATE TABLE statement for table_inventory."""
|
|
66
|
+
return """
|
|
67
|
+
CREATE TABLE IF NOT EXISTS ops.table_inventory (
|
|
68
|
+
inventory_group STRING NOT NULL COMMENT "Group name for a set of tables",
|
|
69
|
+
database_name STRING NOT NULL COMMENT "Database name",
|
|
70
|
+
table_name STRING NOT NULL COMMENT "Table name, or '*' for all tables in database",
|
|
71
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
72
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
73
|
+
)
|
|
74
|
+
PRIMARY KEY (inventory_group, database_name, table_name)
|
|
75
|
+
COMMENT "Inventory groups mapping to databases/tables (supports '*' wildcard)"
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def get_backup_history_schema() -> str:
|
|
80
|
+
"""Get CREATE TABLE statement for backup_history."""
|
|
81
|
+
return """
|
|
82
|
+
CREATE TABLE IF NOT EXISTS ops.backup_history (
|
|
83
|
+
label STRING NOT NULL COMMENT "Unique backup snapshot label",
|
|
84
|
+
backup_type STRING NOT NULL COMMENT "Type of backup: full or incremental",
|
|
85
|
+
status STRING NOT NULL COMMENT "Final backup status: FINISHED, FAILED, CANCELLED, TIMEOUT",
|
|
86
|
+
repository STRING NOT NULL COMMENT "Repository name where backup was stored",
|
|
87
|
+
started_at DATETIME NOT NULL COMMENT "Backup start timestamp",
|
|
88
|
+
finished_at DATETIME COMMENT "Backup completion timestamp",
|
|
89
|
+
error_message STRING COMMENT "Error message if backup failed",
|
|
90
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT "History record creation timestamp"
|
|
91
|
+
)
|
|
92
|
+
PRIMARY KEY (label)
|
|
93
|
+
COMMENT "History log of all backup operations"
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def get_restore_history_schema() -> str:
|
|
98
|
+
"""Get CREATE TABLE statement for restore_history."""
|
|
99
|
+
return """
|
|
100
|
+
CREATE TABLE IF NOT EXISTS ops.restore_history (
|
|
101
|
+
job_id STRING NOT NULL COMMENT "Unique restore job identifier",
|
|
102
|
+
backup_label STRING NOT NULL COMMENT "Source backup snapshot label",
|
|
103
|
+
restore_type STRING NOT NULL COMMENT "Type of restore: partition, table, or database",
|
|
104
|
+
status STRING NOT NULL COMMENT "Final restore status: FINISHED, FAILED, CANCELLED",
|
|
105
|
+
repository STRING NOT NULL COMMENT "Repository name where backup was retrieved from",
|
|
106
|
+
started_at DATETIME NOT NULL COMMENT "Restore start timestamp",
|
|
107
|
+
finished_at DATETIME COMMENT "Restore completion timestamp",
|
|
108
|
+
error_message STRING COMMENT "Error message if restore failed",
|
|
109
|
+
verification_checksum STRING COMMENT "Checksum for data verification",
|
|
110
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT "History record creation timestamp"
|
|
111
|
+
)
|
|
112
|
+
PRIMARY KEY (job_id)
|
|
113
|
+
COMMENT "History log of all restore operations"
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def get_run_status_schema() -> str:
|
|
118
|
+
"""Get CREATE TABLE statement for run_status."""
|
|
119
|
+
return """
|
|
120
|
+
CREATE TABLE IF NOT EXISTS ops.run_status (
|
|
121
|
+
scope STRING NOT NULL COMMENT "Job scope: backup or restore",
|
|
122
|
+
label STRING NOT NULL COMMENT "Job label or identifier",
|
|
123
|
+
state STRING NOT NULL DEFAULT "ACTIVE" COMMENT "Job state: ACTIVE or COMPLETED",
|
|
124
|
+
started_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT "Job start timestamp",
|
|
125
|
+
finished_at DATETIME COMMENT "Job completion timestamp"
|
|
126
|
+
)
|
|
127
|
+
PRIMARY KEY (scope, label)
|
|
128
|
+
COMMENT "Tracks active and recently completed jobs for concurrency control"
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def get_backup_partitions_schema() -> str:
|
|
133
|
+
"""Get CREATE TABLE statement for backup_partitions."""
|
|
134
|
+
return """
|
|
135
|
+
CREATE TABLE IF NOT EXISTS ops.backup_partitions (
|
|
136
|
+
label STRING NOT NULL COMMENT "The backup label this partition belongs to. FK to ops.backup_history.label.",
|
|
137
|
+
database_name STRING NOT NULL COMMENT "The name of the database the partition belongs to.",
|
|
138
|
+
table_name STRING NOT NULL COMMENT "The name of the table the partition belongs to.",
|
|
139
|
+
partition_name STRING NOT NULL COMMENT "The name of the specific partition.",
|
|
140
|
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT "Timestamp when this record was created."
|
|
141
|
+
)
|
|
142
|
+
PRIMARY KEY (label, database_name, table_name, partition_name)
|
|
143
|
+
COMMENT "Tracks every partition included in a backup snapshot."
|
|
144
|
+
"""
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: starrocks-br
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: StarRocks Backup and Restore automation tool
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Requires-Dist: click<9,>=8.1.7
|
|
7
|
+
Requires-Dist: PyYAML<7,>=6.0.1
|
|
8
|
+
Requires-Dist: mysql-connector-python<10,>=9.0.0
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: pytest<9,>=8.3.2; extra == "dev"
|
|
11
|
+
Requires-Dist: pytest-mock<4,>=3.14.0; extra == "dev"
|
|
12
|
+
Requires-Dist: pytest-cov<6,>=5.0.0; extra == "dev"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
starrocks_br/__init__.py,sha256=i1m0FIl2IAXaVyNoya0ZNAx3WfhIp9I6VLhTz06qNFY,28
|
|
2
|
+
starrocks_br/cli.py,sha256=VrLLyvyAop687KNJnXIJhxL7EgVPzutQTAOJz4ANZMw,15969
|
|
3
|
+
starrocks_br/concurrency.py,sha256=wx69u-RW1OnukKn6CQ9EJS4L42N9Gzw7Xz7rgETzmy4,5934
|
|
4
|
+
starrocks_br/config.py,sha256=btmsf43IuxPIraFxJRScuHW-B3Bv7CPO7y43xn2b2cM,1047
|
|
5
|
+
starrocks_br/db.py,sha256=whbkM5LVLNNauPDuSFXOXWuTOFWHBVsNai74O7bIUAg,2450
|
|
6
|
+
starrocks_br/executor.py,sha256=_lbJQ6KjV7Jg5eYbSLZhvvYj-EM5kBV6lM3Lah9h6-U,8748
|
|
7
|
+
starrocks_br/health.py,sha256=DpTy4uqk1UrbV0d9Wtk9Ke9K0iT4ndL-01gqqSywR_c,1050
|
|
8
|
+
starrocks_br/history.py,sha256=j6eqkD1MyTvgoztffnLnr6-6VXd0gdvLxLLKxbC1AG0,3016
|
|
9
|
+
starrocks_br/labels.py,sha256=D67JqIUWtFAnuj9thnC4Y7A0tzLk6d4YpBtDGhen1yc,1689
|
|
10
|
+
starrocks_br/logger.py,sha256=QTfr-nC3TdeU7f1gcRTRDAQSLYpwaevd_iT1B_RbuF8,900
|
|
11
|
+
starrocks_br/planner.py,sha256=wnFMbIQhY9dVaoRjmzB3DypPMEXQam94TFenwGhMGGk,9414
|
|
12
|
+
starrocks_br/repository.py,sha256=6uTJBYgQFEjJBlfhirriTkad80teTPcBSl_tphUExz4,1269
|
|
13
|
+
starrocks_br/restore.py,sha256=IPs8EzIS_7iTUMkKJV32YJvb4b0MTXpLgZmUWhnzPXo,16672
|
|
14
|
+
starrocks_br/schema.py,sha256=_cFzD3Bnvb1WDlLzmFXuMSinLjiXKmLbJ9uUhyOCKK4,6095
|
|
15
|
+
starrocks_br-0.1.0.dist-info/METADATA,sha256=u1ivetyaj0El_v_5Iy9WmJ4_KsEmhU8GUSy3T7Hb4xQ,419
|
|
16
|
+
starrocks_br-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
17
|
+
starrocks_br-0.1.0.dist-info/entry_points.txt,sha256=AKUt01G2MAlh85s1Q9kNQDOUio14kaTnT3dmg9gjdNg,54
|
|
18
|
+
starrocks_br-0.1.0.dist-info/top_level.txt,sha256=CU1tGVo0kjulhDr761Sndg-oTeRKsisDnWm8UG95aBE,13
|
|
19
|
+
starrocks_br-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
starrocks_br
|