starrocks-br 0.5.2__tar.gz → 0.7.0a1__tar.gz

{starrocks_br-0.5.2 → starrocks_br-0.7.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: starrocks-br
-Version: 0.5.2
+Version: 0.7.0a1
 Summary: StarRocks Backup and Restore automation tool
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -56,6 +56,7 @@ This tool adds **incremental backup capabilities** to StarRocks by leveraging na
 - ✅ **Complete operation tracking** - Every backup and restore is logged with status, timestamps, and error details
 - ✅ **Intelligent restore** - Automatically resolves backup chains (full + incremental) for you
 - ✅ **Inventory groups** - Organize tables into groups with different backup strategies
+- ✅ **Backup lifecycle management** - Prune old backups with flexible retention policies (keep-last, older-than, specific snapshots)
 - ✅ **Job concurrency control** - Prevents conflicting operations
 - ✅ **Safe restores** - Atomic rename mechanism prevents data loss during restore
 - ✅ **Metadata management** - Dedicated `ops` database tracks all backup metadata and partition manifests
@@ -95,6 +96,15 @@ port: 9030              # MySQL protocol port
 user: "root"            # Database user with backup/restore privileges
 database: "your_database"   # Database containing tables to backup
 repository: "your_repo_name"  # Repository created via CREATE REPOSITORY in StarRocks
+
+# Optional: Define table inventory groups directly in config
+table_inventory:
+  - group: "production"
+    tables:
+      - database: "mydb"
+        table: "users"
+      - database: "mydb"
+        table: "orders"
 ```
 
 Set password:
@@ -111,7 +121,11 @@ See [Configuration Reference](docs/configuration.md) for TLS and advanced option
 starrocks-br init --config config.yaml
 ```
 
-**Define inventory groups** (in StarRocks):
+This creates the `ops` database and automatically populates table inventory from your config (if defined).
+
+**Note:** If you modify the `table_inventory` in your config file, rerun `starrocks-br init --config config.yaml` to update the database.
+
+**Alternative: Define inventory groups manually** (in StarRocks):
 ```sql
 INSERT INTO ops.table_inventory (inventory_group, database_name, table_name)
 VALUES
@@ -134,6 +148,15 @@ starrocks-br backup incremental --config config.yaml --group production
 starrocks-br restore --config config.yaml --target-label mydb_20251118_full
 ```
 
+**Prune old backups:**
+```bash
+# Keep only last 5 backups
+starrocks-br prune --config config.yaml --keep-last 5
+
+# Delete backups older than a date
+starrocks-br prune --config config.yaml --older-than "2024-01-01 00:00:00"
+```
+
 See [Commands Reference](docs/commands.md) for all options.
 
 ## How It Works

{starrocks_br-0.5.2 → starrocks_br-0.7.0a1}/README.md

@@ -38,6 +38,7 @@ This tool adds **incremental backup capabilities** to StarRocks by leveraging na
 - ✅ **Complete operation tracking** - Every backup and restore is logged with status, timestamps, and error details
 - ✅ **Intelligent restore** - Automatically resolves backup chains (full + incremental) for you
 - ✅ **Inventory groups** - Organize tables into groups with different backup strategies
+- ✅ **Backup lifecycle management** - Prune old backups with flexible retention policies (keep-last, older-than, specific snapshots)
 - ✅ **Job concurrency control** - Prevents conflicting operations
 - ✅ **Safe restores** - Atomic rename mechanism prevents data loss during restore
 - ✅ **Metadata management** - Dedicated `ops` database tracks all backup metadata and partition manifests
@@ -77,6 +78,15 @@ port: 9030              # MySQL protocol port
 user: "root"            # Database user with backup/restore privileges
 database: "your_database"   # Database containing tables to backup
 repository: "your_repo_name"  # Repository created via CREATE REPOSITORY in StarRocks
+
+# Optional: Define table inventory groups directly in config
+table_inventory:
+  - group: "production"
+    tables:
+      - database: "mydb"
+        table: "users"
+      - database: "mydb"
+        table: "orders"
 ```
 
 Set password:
@@ -93,7 +103,11 @@ See [Configuration Reference](docs/configuration.md) for TLS and advanced option
 starrocks-br init --config config.yaml
 ```
 
-**Define inventory groups** (in StarRocks):
+This creates the `ops` database and automatically populates table inventory from your config (if defined).
+
+**Note:** If you modify the `table_inventory` in your config file, rerun `starrocks-br init --config config.yaml` to update the database.
+
+**Alternative: Define inventory groups manually** (in StarRocks):
 ```sql
 INSERT INTO ops.table_inventory (inventory_group, database_name, table_name)
 VALUES
@@ -116,6 +130,15 @@ starrocks-br backup incremental --config config.yaml --group production
 starrocks-br restore --config config.yaml --target-label mydb_20251118_full
 ```
 
+**Prune old backups:**
+```bash
+# Keep only last 5 backups
+starrocks-br prune --config config.yaml --keep-last 5
+
+# Delete backups older than a date
+starrocks-br prune --config config.yaml --older-than "2024-01-01 00:00:00"
+```
+
 See [Commands Reference](docs/commands.md) for all options.
 
 ## How It Works

{starrocks_br-0.5.2 → starrocks_br-0.7.0a1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "starrocks-br"
-version = "0.5.2"
+version = "0.7.0a1"
 description = "StarRocks Backup and Restore automation tool"
 readme = "README.md"
 requires-python = ">=3.10"

{starrocks_br-0.5.2 → starrocks_br-0.7.0a1}/src/starrocks_br/cli.py

@@ -27,6 +27,7 @@ from . import (
     labels,
     logger,
     planner,
+    prune,
     repository,
     restore,
     schema,
@@ -98,13 +99,13 @@ def cli(ctx, verbose):
 @cli.command("init")
 @click.option("--config", required=True, help="Path to config YAML file")
 def init(config):
-    """Initialize ops database and control tables.
+    """Initialize operations database and control tables.
 
-    Creates the ops database with required tables:
-    - ops.table_inventory: Inventory groups mapping to databases/tables
-    - ops.backup_history: Backup operation history
-    - ops.restore_history: Restore operation history
-    - ops.run_status: Job concurrency control
+    Creates the operations database (default: 'ops') with required tables:
+    - table_inventory: Inventory groups mapping to databases/tables
+    - backup_history: Backup operation history
+    - restore_history: Restore operation history
+    - run_status: Job concurrency control
 
     Run this once before using backup/restore commands.
     """
@@ -112,6 +113,9 @@ def init(config):
         cfg = config_module.load_config(config)
         config_module.validate_config(cfg)
 
+        ops_database = config_module.get_ops_database(cfg)
+        table_inventory_entries = config_module.get_table_inventory_entries(cfg)
+
         database = db.StarRocksDB(
             host=cfg["host"],
             port=cfg["port"],
@@ -121,23 +125,43 @@ def init(config):
             tls_config=cfg.get("tls"),
         )
 
+        ops_database = config_module.get_ops_database(cfg)
+
         with database:
-            logger.info("Initializing ops schema...")
-            schema.initialize_ops_schema(database)
-            logger.info("")
-            logger.info("Next steps:")
-            logger.info("1. Insert your table inventory records:")
-            logger.info("   INSERT INTO ops.table_inventory")
-            logger.info("   (inventory_group, database_name, table_name)")
-            logger.info("   VALUES ('my_daily_incremental', 'your_db', 'your_fact_table');")
-            logger.info("   VALUES ('my_full_database_backup', 'your_db', '*');")
-            logger.info("   VALUES ('my_full_dimension_tables', 'your_db', 'dim_customers');")
-            logger.info("   VALUES ('my_full_dimension_tables', 'your_db', 'dim_products');")
+            logger.info("Validating repository...")
+            repository.ensure_repository(database, cfg["repository"])
             logger.info("")
-            logger.info("2. Run your first backup:")
-            logger.info(
-                "   starrocks-br backup incremental --group my_daily_incremental --config config.yaml"
+
+            logger.info("Initializing ops schema...")
+            schema.initialize_ops_schema(
+                database, ops_database=ops_database, table_inventory_entries=table_inventory_entries
             )
+            logger.info("")
+
+            if table_inventory_entries:
+                logger.success(
+                    f"Table inventory bootstrapped from config with {len(table_inventory_entries)} entries"
+                )
+                logger.info("")
+                logger.info("Next steps:")
+                logger.info("1. Run your first backup:")
+                logger.info(
+                    f"   starrocks-br backup incremental --group <your_group_name> --config {config}"
+                )
+            else:
+                logger.info("Next steps:")
+                logger.info("1. Insert your table inventory records:")
+                logger.info(f"   INSERT INTO {ops_database}.table_inventory")
+                logger.info("   (inventory_group, database_name, table_name)")
+                logger.info("   VALUES ('my_daily_incremental', 'your_db', 'your_fact_table');")
+                logger.info("   VALUES ('my_full_database_backup', 'your_db', '*');")
+                logger.info("   VALUES ('my_full_dimension_tables', 'your_db', 'dim_customers');")
+                logger.info("   VALUES ('my_full_dimension_tables', 'your_db', 'dim_products');")
+                logger.info("")
+                logger.info("2. Run your first backup:")
+                logger.info(
+                    "   starrocks-br backup incremental --group my_daily_incremental --config config.yaml"
+                )
 
     except exceptions.ConfigFileNotFoundError as e:
         error_handler.handle_config_file_not_found_error(e)
@@ -196,13 +220,17 @@ def backup_incremental(config, baseline_backup, group, name):
             tls_config=cfg.get("tls"),
         )
 
+        ops_database = config_module.get_ops_database(cfg)
+
         with database:
-            was_created = schema.ensure_ops_schema(database)
+            was_created = schema.ensure_ops_schema(database, ops_database=ops_database)
             if was_created:
                 logger.warning(
                     "ops schema was auto-created. Please run 'starrocks-br init' after populating config."
                 )
-                logger.warning("Remember to populate ops.table_inventory with your backup groups!")
+                logger.warning(
+                    "Remember to populate the table_inventory table with your backup groups!"
+                )
                 sys.exit(1)  # Exit if schema was just created, requires user action
 
             healthy, message = health.check_cluster_health(database)
@@ -221,6 +249,7 @@ def backup_incremental(config, baseline_backup, group, name):
                 backup_type="incremental",
                 database_name=cfg["database"],
                 custom_name=name,
+                ops_database=ops_database,
             )
 
             logger.success(f"Generated label: {label}")
@@ -239,7 +268,11 @@ def backup_incremental(config, baseline_backup, group, name):
                     )
 
             partitions = planner.find_recent_partitions(
-                database, cfg["database"], baseline_backup_label=baseline_backup, group_name=group
+                database,
+                cfg["database"],
+                baseline_backup_label=baseline_backup,
+                group_name=group,
+                ops_database=ops_database,
             )
 
             if not partitions:
@@ -252,9 +285,11 @@ def backup_incremental(config, baseline_backup, group, name):
                 partitions, cfg["repository"], label, cfg["database"]
             )
 
-            concurrency.reserve_job_slot(database, scope="backup", label=label)
+            concurrency.reserve_job_slot(
+                database, scope="backup", label=label, ops_database=ops_database
+            )
 
-            planner.record_backup_partitions(database, label, partitions)
+            planner.record_backup_partitions(database, label, partitions, ops_database=ops_database)
 
             logger.success("Job slot reserved")
             logger.info(f"Starting incremental backup for group '{group}'...")
@@ -265,6 +300,7 @@ def backup_incremental(config, baseline_backup, group, name):
                 backup_type="incremental",
                 scope="backup",
                 database=cfg["database"],
+                ops_database=ops_database,
             )
 
             if result["success"]:
@@ -288,7 +324,7 @@ def backup_incremental(config, baseline_backup, group, name):
                 if state == "LOST":
                     logger.critical("Backup tracking lost!")
                     logger.warning("Another backup operation started during ours.")
-                    logger.tip("Enable ops.run_status concurrency checks to prevent this.")
+                    logger.tip("Enable run_status concurrency checks to prevent this.")
                 logger.error(f"{result['error_message']}")
                 sys.exit(1)
 
@@ -348,13 +384,17 @@ def backup_full(config, group, name):
             tls_config=cfg.get("tls"),
         )
 
+        ops_database = config_module.get_ops_database(cfg)
+
         with database:
-            was_created = schema.ensure_ops_schema(database)
+            was_created = schema.ensure_ops_schema(database, ops_database=ops_database)
             if was_created:
                 logger.warning(
                     "ops schema was auto-created. Please run 'starrocks-br init' after populating config."
                 )
-                logger.warning("Remember to populate ops.table_inventory with your backup groups!")
+                logger.warning(
+                    "Remember to populate the table_inventory table with your backup groups!"
+                )
                 sys.exit(1)  # Exit if schema was just created, requires user action
 
             healthy, message = health.check_cluster_health(database)
@@ -369,13 +409,25 @@ def backup_full(config, group, name):
             logger.success(f"Repository '{cfg['repository']}' verified")
 
             label = labels.determine_backup_label(
-                db=database, backup_type="full", database_name=cfg["database"], custom_name=name
+                db=database,
+                backup_type="full",
+                database_name=cfg["database"],
+                custom_name=name,
+                ops_database=ops_database,
             )
 
             logger.success(f"Generated label: {label}")
 
+            tables = planner.find_tables_by_group(database, group, ops_database)
+            planner.validate_tables_exist(database, cfg["database"], tables, group)
+
             backup_command = planner.build_full_backup_command(
-                database, group, cfg["repository"], label, cfg["database"]
+                database,
+                group,
+                cfg["repository"],
+                label,
+                cfg["database"],
+                ops_database=ops_database,
             )
 
             if not backup_command:
@@ -389,9 +441,13 @@ def backup_full(config, group, name):
                 database, cfg["database"], tables
             )
 
-            concurrency.reserve_job_slot(database, scope="backup", label=label)
+            concurrency.reserve_job_slot(
+                database, scope="backup", label=label, ops_database=ops_database
+            )
 
-            planner.record_backup_partitions(database, label, all_partitions)
+            planner.record_backup_partitions(
+                database, label, all_partitions, ops_database=ops_database
+            )
 
             logger.success("Job slot reserved")
             logger.info(f"Starting full backup for group '{group}'...")
@@ -402,6 +458,7 @@ def backup_full(config, group, name):
                 backup_type="full",
                 scope="backup",
                 database=cfg["database"],
+                ops_database=ops_database,
             )
 
             if result["success"]:
@@ -419,10 +476,13 @@ def backup_full(config, group, name):
                 if state == "LOST":
                     logger.critical("Backup tracking lost!")
                     logger.warning("Another backup operation started during ours.")
-                    logger.tip("Enable ops.run_status concurrency checks to prevent this.")
+                    logger.tip("Enable run_status concurrency checks to prevent this.")
                 logger.error(f"{result['error_message']}")
                 sys.exit(1)
 
+    except exceptions.InvalidTablesInInventoryError as e:
+        error_handler.handle_invalid_tables_in_inventory_error(e, config)
+        sys.exit(1)
     except exceptions.ConcurrencyConflictError as e:
         error_handler.handle_concurrency_conflict_error(e, config)
         sys.exit(1)
@@ -499,13 +559,17 @@ def restore_command(config, target_label, group, table, rename_suffix, yes):
             tls_config=cfg.get("tls"),
         )
 
+        ops_database = config_module.get_ops_database(cfg)
+
         with database:
-            was_created = schema.ensure_ops_schema(database)
+            was_created = schema.ensure_ops_schema(database, ops_database=ops_database)
             if was_created:
                 logger.warning(
                     "ops schema was auto-created. Please run 'starrocks-br init' after populating config."
                 )
-                logger.warning("Remember to populate ops.table_inventory with your backup groups!")
+                logger.warning(
+                    "Remember to populate the table_inventory table with your backup groups!"
+                )
                 sys.exit(1)  # Exit if schema was just created, requires user action
 
             healthy, message = health.check_cluster_health(database)
@@ -521,7 +585,9 @@ def restore_command(config, target_label, group, table, rename_suffix, yes):
 
             logger.info(f"Finding restore sequence for target backup: {target_label}")
 
-            restore_pair = restore.find_restore_pair(database, target_label)
+            restore_pair = restore.find_restore_pair(
+                database, target_label, ops_database=ops_database
+            )
             logger.success(f"Found restore sequence: {' -> '.join(restore_pair)}")
 
             logger.info("Determining tables to restore from backup manifest...")
@@ -532,6 +598,7 @@ def restore_command(config, target_label, group, table, rename_suffix, yes):
                 group=group,
                 table=table,
                 database=cfg["database"] if table else None,
+                ops_database=ops_database,
             )
 
             if not tables_to_restore:
@@ -549,6 +616,7 @@ def restore_command(config, target_label, group, table, rename_suffix, yes):
                 tables_to_restore,
                 rename_suffix,
                 skip_confirmation=yes,
+                ops_database=ops_database,
             )
 
             if result["success"]:
@@ -604,5 +672,209 @@ def restore_command(config, target_label, group, table, rename_suffix, yes):
         sys.exit(1)
 
 
+@cli.command("prune")
+@click.option("--config", required=True, help="Path to config YAML file")
+@click.option(
+    "--group",
+    help="Optional inventory group to filter backups. Without this, prunes ALL backups.",
+)
+@click.option(
+    "--keep-last",
+    type=int,
+    help="Keep only the last N successful backups (deletes older ones)",
+)
+@click.option(
+    "--older-than",
+    help="Delete snapshots older than this timestamp (format: YYYY-MM-DD HH:MM:SS)",
+)
+@click.option("--snapshot", help="Delete a specific snapshot by name")
+@click.option("--snapshots", help="Delete multiple specific snapshots (comma-separated)")
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Show what would be deleted without actually deleting",
+)
+@click.option("--yes", is_flag=True, help="Skip confirmation prompt and proceed automatically")
+def prune_command(config, group, keep_last, older_than, snapshot, snapshots, dry_run, yes):
+    """Prune (delete) old backup snapshots from the repository.
+
+    This command helps manage repository storage by removing old or unwanted snapshots.
+    Supports multiple pruning strategies:
+    - Keep only the last N backups
+    - Delete backups older than a specific date
+    - Delete specific snapshots by name
+
+    Flow: load config → check health → ensure repository → query backups →
+    filter snapshots to delete → confirm → execute deletion → cleanup history
+    """
+    try:
+        pruning_options = [keep_last, older_than, snapshot, snapshots]
+        specified_options = [opt for opt in pruning_options if opt is not None]
+
+        if not specified_options:
+            logger.error(
+                "Must specify one pruning option: --keep-last, --older-than, --snapshot, or --snapshots"
+            )
+            sys.exit(1)
+
+        if len(specified_options) > 1:
+            logger.error(
+                "Pruning options are mutually exclusive. "
+                "Please specify only one of: --keep-last, --older-than, --snapshot, or --snapshots"
+            )
+            sys.exit(1)
+
+        if keep_last is not None and keep_last <= 0:
+            logger.error("--keep-last must be a positive number (greater than 0)")
+            sys.exit(1)
+
+        cfg = config_module.load_config(config)
+        config_module.validate_config(cfg)
+
+        database = db.StarRocksDB(
+            host=cfg["host"],
+            port=cfg["port"],
+            user=cfg["user"],
+            password=os.getenv("STARROCKS_PASSWORD"),
+            database=cfg["database"],
+            tls_config=cfg.get("tls"),
+        )
+
+        ops_database = config_module.get_ops_database(cfg)
+
+        with database:
+            was_created = schema.ensure_ops_schema(database, ops_database=ops_database)
+            if was_created:
+                logger.warning(
+                    "ops schema was auto-created. Please run 'starrocks-br init' after populating config."
+                )
+                sys.exit(1)
+
+            healthy, message = health.check_cluster_health(database)
+            if not healthy:
+                logger.error(f"Cluster health check failed: {message}")
+                sys.exit(1)
+
+            logger.success(f"Cluster health: {message}")
+
+            repository.ensure_repository(database, cfg["repository"])
+            logger.success(f"Repository '{cfg['repository']}' verified")
+
+            if keep_last:
+                strategy = "keep_last"
+                strategy_kwargs = {"count": keep_last}
+                logger.info(f"Pruning strategy: Keep last {keep_last} backup(s)")
+            elif older_than:
+                strategy = "older_than"
+                strategy_kwargs = {"timestamp": older_than}
+                logger.info(f"Pruning strategy: Delete backups older than {older_than}")
+            elif snapshot:
+                strategy = "specific"
+                strategy_kwargs = {"snapshot": snapshot}
+                logger.info(f"Pruning strategy: Delete specific snapshot '{snapshot}'")
+            elif snapshots:
+                strategy = "multiple"
+                snapshot_list = [s.strip() for s in snapshots.split(",")]
+                strategy_kwargs = {"snapshots": snapshot_list}
+                logger.info(f"Pruning strategy: Delete {len(snapshot_list)} specific snapshot(s)")
+
+            if group:
+                logger.info(f"Filtering by inventory group: {group}")
+
+            all_backups = prune.get_successful_backups(
+                database, cfg["repository"], group=group, ops_database=ops_database
+            )
+
+            if not all_backups:
+                msg = f"No successful backups found in repository '{cfg['repository']}'"
+                if group:
+                    msg += f" for inventory group '{group}'"
+                logger.info(msg)
+                sys.exit(0)
+
+            logger.info(f"Found {len(all_backups)} total backup(s) in repository")
+
+            if strategy in ["specific", "multiple"]:
+                snapshots_to_verify = (
+                    [snapshot] if strategy == "specific" else strategy_kwargs["snapshots"]
+                )
+                for snap in snapshots_to_verify:
+                    prune.verify_snapshot_exists(database, cfg["repository"], snap)
+
+            snapshots_to_delete = prune.filter_snapshots_to_delete(
+                all_backups, strategy, **strategy_kwargs
+            )
+
+            if not snapshots_to_delete:
+                logger.success("No snapshots to delete based on the specified criteria")
+                sys.exit(0)
+
+            logger.info("")
+            logger.info(f"Snapshots to delete: {len(snapshots_to_delete)}")
+            for snap in snapshots_to_delete:
+                logger.info(f"  - {snap['label']} (finished: {snap['finished_at']})")
+
+            if keep_last:
+                kept_count = len(all_backups) - len(snapshots_to_delete)
+                logger.info(f"Snapshots to keep: {kept_count} (most recent)")
+
+            if dry_run:
+                logger.info("")
+                logger.warning("DRY RUN MODE - No snapshots will be deleted")
+                logger.info(f"Would delete {len(snapshots_to_delete)} snapshot(s)")
+                sys.exit(0)
+
+            if not yes:
+                logger.info("")
+                logger.warning(
+                    f"This will permanently delete {len(snapshots_to_delete)} snapshot(s) from the repository"
+                )
+                confirm = click.confirm("Do you want to proceed?", default=False)
+                if not confirm:
+                    logger.info("Prune operation cancelled by user")
+                    sys.exit(1)
+
+            logger.info("")
+            logger.info("Starting snapshot deletion...")
+            deleted_count = 0
+            failed_count = 0
+
+            for snap in snapshots_to_delete:
+                try:
+                    prune.execute_drop_snapshot(database, cfg["repository"], snap["label"])
+                    prune.cleanup_backup_history(database, snap["label"], ops_database=ops_database)
+                    deleted_count += 1
+                except Exception as e:
+                    logger.error(f"Failed to delete snapshot '{snap['label']}': {e}")
+                    failed_count += 1
+
+            logger.info("")
+            logger.success(f"Deleted {deleted_count} snapshot(s)")
+
+            if failed_count > 0:
+                logger.warning(f"Failed to delete {failed_count} snapshot(s)")
+
+            if keep_last:
+                logger.success(f"Kept {len(all_backups) - len(snapshots_to_delete)} most recent backup(s)")
+
+            sys.exit(0 if failed_count == 0 else 1)
+
+    except exceptions.ConfigFileNotFoundError as e:
+        error_handler.handle_config_file_not_found_error(e)
+        sys.exit(1)
+    except exceptions.ConfigValidationError as e:
+        error_handler.handle_config_validation_error(e, config)
+        sys.exit(1)
+    except FileNotFoundError as e:
+        error_handler.handle_config_file_not_found_error(exceptions.ConfigFileNotFoundError(str(e)))
+        sys.exit(1)
+    except ValueError as e:
+        logger.error(f"Validation error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        sys.exit(1)
+
+
 if __name__ == "__main__":
     cli()