starrocks-br 0.6.0__tar.gz → 0.7.0a1__tar.gz

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: starrocks-br
-Version: 0.6.0
+Version: 0.7.0a1
 Summary: StarRocks Backup and Restore automation tool
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -56,6 +56,7 @@ This tool adds **incremental backup capabilities** to StarRocks by leveraging na
 - ✅ **Complete operation tracking** - Every backup and restore is logged with status, timestamps, and error details
 - ✅ **Intelligent restore** - Automatically resolves backup chains (full + incremental) for you
 - ✅ **Inventory groups** - Organize tables into groups with different backup strategies
+- ✅ **Backup lifecycle management** - Prune old backups with flexible retention policies (keep-last, older-than, specific snapshots)
 - ✅ **Job concurrency control** - Prevents conflicting operations
 - ✅ **Safe restores** - Atomic rename mechanism prevents data loss during restore
 - ✅ **Metadata management** - Dedicated `ops` database tracks all backup metadata and partition manifests
@@ -147,6 +148,15 @@ starrocks-br backup incremental --config config.yaml --group production
 starrocks-br restore --config config.yaml --target-label mydb_20251118_full
 ```
 
+**Prune old backups:**
+```bash
+# Keep only last 5 backups
+starrocks-br prune --config config.yaml --keep-last 5
+
+# Delete backups older than a date
+starrocks-br prune --config config.yaml --older-than "2024-01-01 00:00:00"
+```
+
 See [Commands Reference](docs/commands.md) for all options.
 
 ## How It Works

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/README.md

@@ -38,6 +38,7 @@ This tool adds **incremental backup capabilities** to StarRocks by leveraging na
 - ✅ **Complete operation tracking** - Every backup and restore is logged with status, timestamps, and error details
 - ✅ **Intelligent restore** - Automatically resolves backup chains (full + incremental) for you
 - ✅ **Inventory groups** - Organize tables into groups with different backup strategies
+- ✅ **Backup lifecycle management** - Prune old backups with flexible retention policies (keep-last, older-than, specific snapshots)
 - ✅ **Job concurrency control** - Prevents conflicting operations
 - ✅ **Safe restores** - Atomic rename mechanism prevents data loss during restore
 - ✅ **Metadata management** - Dedicated `ops` database tracks all backup metadata and partition manifests
@@ -129,6 +130,15 @@ starrocks-br backup incremental --config config.yaml --group production
 starrocks-br restore --config config.yaml --target-label mydb_20251118_full
 ```
 
+**Prune old backups:**
+```bash
+# Keep only last 5 backups
+starrocks-br prune --config config.yaml --keep-last 5
+
+# Delete backups older than a date
+starrocks-br prune --config config.yaml --older-than "2024-01-01 00:00:00"
+```
+
 See [Commands Reference](docs/commands.md) for all options.
 
 ## How It Works

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "starrocks-br"
-version = "0.6.0"
+version = "0.7.0a1"
 description = "StarRocks Backup and Restore automation tool"
 readme = "README.md"
 requires-python = ">=3.10"

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/src/starrocks_br/cli.py

@@ -27,6 +27,7 @@ from . import (
     labels,
     logger,
     planner,
+    prune,
     repository,
     restore,
     schema,
@@ -671,5 +672,209 @@ def restore_command(config, target_label, group, table, rename_suffix, yes):
         sys.exit(1)
 
 
+@cli.command("prune")
+@click.option("--config", required=True, help="Path to config YAML file")
+@click.option(
+    "--group",
+    help="Optional inventory group to filter backups. Without this, prunes ALL backups.",
+)
+@click.option(
+    "--keep-last",
+    type=int,
+    help="Keep only the last N successful backups (deletes older ones)",
+)
+@click.option(
+    "--older-than",
+    help="Delete snapshots older than this timestamp (format: YYYY-MM-DD HH:MM:SS)",
+)
+@click.option("--snapshot", help="Delete a specific snapshot by name")
+@click.option("--snapshots", help="Delete multiple specific snapshots (comma-separated)")
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Show what would be deleted without actually deleting",
+)
+@click.option("--yes", is_flag=True, help="Skip confirmation prompt and proceed automatically")
+def prune_command(config, group, keep_last, older_than, snapshot, snapshots, dry_run, yes):
+    """Prune (delete) old backup snapshots from the repository.
+
+    This command helps manage repository storage by removing old or unwanted snapshots.
+    Supports multiple pruning strategies:
+    - Keep only the last N backups
+    - Delete backups older than a specific date
+    - Delete specific snapshots by name
+
+    Flow: load config → check health → ensure repository → query backups →
+    filter snapshots to delete → confirm → execute deletion → cleanup history
+    """
+    try:
+        pruning_options = [keep_last, older_than, snapshot, snapshots]
+        specified_options = [opt for opt in pruning_options if opt is not None]
+
+        if not specified_options:
+            logger.error(
+                "Must specify one pruning option: --keep-last, --older-than, --snapshot, or --snapshots"
+            )
+            sys.exit(1)
+
+        if len(specified_options) > 1:
+            logger.error(
+                "Pruning options are mutually exclusive. "
+                "Please specify only one of: --keep-last, --older-than, --snapshot, or --snapshots"
+            )
+            sys.exit(1)
+
+        if keep_last is not None and keep_last <= 0:
+            logger.error("--keep-last must be a positive number (greater than 0)")
+            sys.exit(1)
+
+        cfg = config_module.load_config(config)
+        config_module.validate_config(cfg)
+
+        database = db.StarRocksDB(
+            host=cfg["host"],
+            port=cfg["port"],
+            user=cfg["user"],
+            password=os.getenv("STARROCKS_PASSWORD"),
+            database=cfg["database"],
+            tls_config=cfg.get("tls"),
+        )
+
+        ops_database = config_module.get_ops_database(cfg)
+
+        with database:
+            was_created = schema.ensure_ops_schema(database, ops_database=ops_database)
+            if was_created:
+                logger.warning(
+                    "ops schema was auto-created. Please run 'starrocks-br init' after populating config."
+                )
+                sys.exit(1)
+
+            healthy, message = health.check_cluster_health(database)
+            if not healthy:
+                logger.error(f"Cluster health check failed: {message}")
+                sys.exit(1)
+
+            logger.success(f"Cluster health: {message}")
+
+            repository.ensure_repository(database, cfg["repository"])
+            logger.success(f"Repository '{cfg['repository']}' verified")
+
+            if keep_last:
+                strategy = "keep_last"
+                strategy_kwargs = {"count": keep_last}
+                logger.info(f"Pruning strategy: Keep last {keep_last} backup(s)")
+            elif older_than:
+                strategy = "older_than"
+                strategy_kwargs = {"timestamp": older_than}
+                logger.info(f"Pruning strategy: Delete backups older than {older_than}")
+            elif snapshot:
+                strategy = "specific"
+                strategy_kwargs = {"snapshot": snapshot}
+                logger.info(f"Pruning strategy: Delete specific snapshot '{snapshot}'")
+            elif snapshots:
+                strategy = "multiple"
+                snapshot_list = [s.strip() for s in snapshots.split(",")]
+                strategy_kwargs = {"snapshots": snapshot_list}
+                logger.info(f"Pruning strategy: Delete {len(snapshot_list)} specific snapshot(s)")
+
+            if group:
+                logger.info(f"Filtering by inventory group: {group}")
+
+            all_backups = prune.get_successful_backups(
+                database, cfg["repository"], group=group, ops_database=ops_database
+            )
+
+            if not all_backups:
+                msg = f"No successful backups found in repository '{cfg['repository']}'"
+                if group:
+                    msg += f" for inventory group '{group}'"
+                logger.info(msg)
+                sys.exit(0)
+
+            logger.info(f"Found {len(all_backups)} total backup(s) in repository")
+
+            if strategy in ["specific", "multiple"]:
+                snapshots_to_verify = (
+                    [snapshot] if strategy == "specific" else strategy_kwargs["snapshots"]
+                )
+                for snap in snapshots_to_verify:
+                    prune.verify_snapshot_exists(database, cfg["repository"], snap)
+
+            snapshots_to_delete = prune.filter_snapshots_to_delete(
+                all_backups, strategy, **strategy_kwargs
+            )
+
+            if not snapshots_to_delete:
+                logger.success("No snapshots to delete based on the specified criteria")
+                sys.exit(0)
+
+            logger.info("")
+            logger.info(f"Snapshots to delete: {len(snapshots_to_delete)}")
+            for snap in snapshots_to_delete:
+                logger.info(f"  - {snap['label']} (finished: {snap['finished_at']})")
+
+            if keep_last:
+                kept_count = len(all_backups) - len(snapshots_to_delete)
+                logger.info(f"Snapshots to keep: {kept_count} (most recent)")
+
+            if dry_run:
+                logger.info("")
+                logger.warning("DRY RUN MODE - No snapshots will be deleted")
+                logger.info(f"Would delete {len(snapshots_to_delete)} snapshot(s)")
+                sys.exit(0)
+
+            if not yes:
+                logger.info("")
+                logger.warning(
+                    f"This will permanently delete {len(snapshots_to_delete)} snapshot(s) from the repository"
+                )
+                confirm = click.confirm("Do you want to proceed?", default=False)
+                if not confirm:
+                    logger.info("Prune operation cancelled by user")
+                    sys.exit(1)
+
+            logger.info("")
+            logger.info("Starting snapshot deletion...")
+            deleted_count = 0
+            failed_count = 0
+
+            for snap in snapshots_to_delete:
+                try:
+                    prune.execute_drop_snapshot(database, cfg["repository"], snap["label"])
+                    prune.cleanup_backup_history(database, snap["label"], ops_database=ops_database)
+                    deleted_count += 1
+                except Exception as e:
+                    logger.error(f"Failed to delete snapshot '{snap['label']}': {e}")
+                    failed_count += 1
+
+            logger.info("")
+            logger.success(f"Deleted {deleted_count} snapshot(s)")
+
+            if failed_count > 0:
+                logger.warning(f"Failed to delete {failed_count} snapshot(s)")
+
+            if keep_last:
+                logger.success(f"Kept {len(all_backups) - len(snapshots_to_delete)} most recent backup(s)")
+
+            sys.exit(0 if failed_count == 0 else 1)
+
+    except exceptions.ConfigFileNotFoundError as e:
+        error_handler.handle_config_file_not_found_error(e)
+        sys.exit(1)
+    except exceptions.ConfigValidationError as e:
+        error_handler.handle_config_validation_error(e, config)
+        sys.exit(1)
+    except FileNotFoundError as e:
+        error_handler.handle_config_file_not_found_error(exceptions.ConfigFileNotFoundError(str(e)))
+        sys.exit(1)
+    except ValueError as e:
+        logger.error(f"Validation error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        sys.exit(1)
+
+
 if __name__ == "__main__":
     cli()

starrocks_br-0.7.0a1/src/starrocks_br/prune.py

@@ -0,0 +1,208 @@
+# Copyright 2025 deep-bi
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import datetime
+
+from . import logger
+
+
+def get_successful_backups(
+    db, repository: str, group: str = None, ops_database: str = "ops"
+) -> list[dict]:
+    """Get all successful backups from backup_history, optionally filtered by group.
+
+    Args:
+        db: Database connection
+        repository: Repository name to filter by
+        group: Optional inventory group to filter by
+        ops_database: Name of the ops database (defaults to "ops")
+
+    Returns:
+        List of backup records as dicts with keys: label, finished_at, inventory_group (if group filtering is used)
+    """
+    if group:
+        sql = f"""
+        SELECT DISTINCT
+            bh.label,
+            bh.finished_at,
+            ti.inventory_group
+        FROM {ops_database}.backup_history bh
+        INNER JOIN {ops_database}.backup_partitions bp ON bh.label = bp.label
+        INNER JOIN {ops_database}.table_inventory ti
+            ON bp.database_name = ti.database_name
+            AND (bp.table_name = ti.table_name OR ti.table_name = '*')
+        WHERE bh.repository = '{repository}'
+            AND bh.status = 'FINISHED'
+            AND ti.inventory_group = '{group}'
+        ORDER BY bh.finished_at ASC
+        """
+    else:
+        sql = f"""
+        SELECT
+            label,
+            finished_at
+        FROM {ops_database}.backup_history
+        WHERE repository = '{repository}'
+            AND status = 'FINISHED'
+        ORDER BY finished_at ASC
+        """
+
+    rows = db.query(sql)
+    results = []
+
+    for row in rows:
+        if group:
+            results.append({"label": row[0], "finished_at": str(row[1]), "inventory_group": row[2]})
+        else:
+            results.append({"label": row[0], "finished_at": str(row[1])})
+
+    return results
+
+
+def filter_snapshots_to_delete(
+    all_snapshots: list[dict], strategy: str, **kwargs
+) -> list[dict]:
+    """Filter snapshots based on pruning strategy.
+
+    Args:
+        all_snapshots: List of snapshot dicts (must be sorted by finished_at ASC)
+        strategy: Pruning strategy - 'keep_last', 'older_than', 'specific', or 'multiple'
+        **kwargs: Strategy-specific parameters:
+            - keep_last: 'count' (int) - number of backups to keep
+            - older_than: 'timestamp' (str) - timestamp in 'YYYY-MM-DD HH:MM:SS' format
+            - specific: 'snapshot' (str) - specific snapshot name
+            - multiple: 'snapshots' (list) - list of snapshot names
+
+    Returns:
+        List of snapshots to delete
+    """
+    if strategy == "keep_last":
+        count = kwargs.get("count")
+        if count is None or count <= 0:
+            raise ValueError("keep_last strategy requires a positive count")
+
+        # Keep the last N, delete the rest
+        if len(all_snapshots) <= count:
+            return []
+        return all_snapshots[: -count]  # Delete all except last N
+
+    elif strategy == "older_than":
+        timestamp_str = kwargs.get("timestamp")
+        if not timestamp_str:
+            raise ValueError("older_than strategy requires a timestamp")
+
+        try:
+            cutoff = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S")
+        except ValueError as e:
+            raise ValueError(
+                f"Invalid timestamp format '{timestamp_str}'. Expected 'YYYY-MM-DD HH:MM:SS'"
+            ) from e
+
+        to_delete = []
+        for snapshot in all_snapshots:
+            snapshot_time = datetime.strptime(snapshot["finished_at"], "%Y-%m-%d %H:%M:%S")
+            if snapshot_time < cutoff:
+                to_delete.append(snapshot)
+
+        return to_delete
+
+    elif strategy == "specific":
+        snapshot_name = kwargs.get("snapshot")
+        if not snapshot_name:
+            raise ValueError("specific strategy requires a snapshot name")
+
+        for snapshot in all_snapshots:
+            if snapshot["label"] == snapshot_name:
+                return [snapshot]
+
+        return []
+
+    elif strategy == "multiple":
+        snapshot_names = kwargs.get("snapshots")
+        if not snapshot_names:
+            raise ValueError("multiple strategy requires a list of snapshot names")
+
+        to_delete = []
+        for snapshot in all_snapshots:
+            if snapshot["label"] in snapshot_names:
+                to_delete.append(snapshot)
+
+        return to_delete
+
+    else:
+        raise ValueError(f"Unknown pruning strategy: {strategy}")
+
+
+def verify_snapshot_exists(db, repository: str, snapshot_name: str) -> bool:
+    """Verify that a snapshot exists in the repository.
+
+    Args:
+        db: Database connection
+        repository: Repository name
+        snapshot_name: Snapshot name to verify
+
+    Returns:
+        True if snapshot exists, False otherwise
+
+    Raises:
+        Exception if snapshot is not found
+    """
+    sql = f"SHOW SNAPSHOT ON {repository} WHERE SNAPSHOT = '{snapshot_name}'"
+
+    try:
+        rows = db.query(sql)
+        if not rows:
+            raise Exception(f"Snapshot '{snapshot_name}' not found in repository '{repository}'")
+        return True
+    except Exception as e:
+        logger.error(f"Failed to verify snapshot '{snapshot_name}': {e}")
+        raise
+
+
+def execute_drop_snapshot(db, repository: str, snapshot_name: str) -> None:
+    """Execute DROP SNAPSHOT command for a single snapshot.
+
+    Args:
+        db: Database connection
+        repository: Repository name
+        snapshot_name: Snapshot name to delete
+
+    Raises:
+        Exception if deletion fails
+    """
+    sql = f"DROP SNAPSHOT ON {repository} WHERE SNAPSHOT = '{snapshot_name}'"
+
+    try:
+        logger.info(f"Deleting snapshot: {snapshot_name}")
+        db.execute(sql)
+        logger.success(f"Successfully deleted snapshot: {snapshot_name}")
+    except Exception as e:
+        logger.error(f"Failed to delete snapshot '{snapshot_name}': {e}")
+        raise
+
+
+def cleanup_backup_history(db, snapshot_label: str, ops_database: str = "ops") -> None:
+    """Remove backup history entry after snapshot deletion.
+
+    Args:
+        db: Database connection
+        snapshot_label: Snapshot label to remove from history
+        ops_database: Name of the ops database (defaults to "ops")
+    """
+    try:
+        db.execute(f"DELETE FROM {ops_database}.backup_partitions WHERE label = '{snapshot_label}'")
+        db.execute(f"DELETE FROM {ops_database}.backup_history WHERE label = '{snapshot_label}'")
+        logger.debug(f"Cleaned up backup history for: {snapshot_label}")
+    except Exception as e:
+        logger.warning(f"Failed to cleanup backup history for '{snapshot_label}': {e}")

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/src/starrocks_br.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: starrocks-br
-Version: 0.6.0
+Version: 0.7.0a1
 Summary: StarRocks Backup and Restore automation tool
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
@@ -56,6 +56,7 @@ This tool adds **incremental backup capabilities** to StarRocks by leveraging na
 - ✅ **Complete operation tracking** - Every backup and restore is logged with status, timestamps, and error details
 - ✅ **Intelligent restore** - Automatically resolves backup chains (full + incremental) for you
 - ✅ **Inventory groups** - Organize tables into groups with different backup strategies
+- ✅ **Backup lifecycle management** - Prune old backups with flexible retention policies (keep-last, older-than, specific snapshots)
 - ✅ **Job concurrency control** - Prevents conflicting operations
 - ✅ **Safe restores** - Atomic rename mechanism prevents data loss during restore
 - ✅ **Metadata management** - Dedicated `ops` database tracks all backup metadata and partition manifests
@@ -147,6 +148,15 @@ starrocks-br backup incremental --config config.yaml --group production
 starrocks-br restore --config config.yaml --target-label mydb_20251118_full
 ```
 
+**Prune old backups:**
+```bash
+# Keep only last 5 backups
+starrocks-br prune --config config.yaml --keep-last 5
+
+# Delete backups older than a date
+starrocks-br prune --config config.yaml --older-than "2024-01-01 00:00:00"
+```
+
 See [Commands Reference](docs/commands.md) for all options.
 
 ## How It Works

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/src/starrocks_br.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ src/starrocks_br/history.py
 src/starrocks_br/labels.py
 src/starrocks_br/logger.py
 src/starrocks_br/planner.py
+src/starrocks_br/prune.py
 src/starrocks_br/repository.py
 src/starrocks_br/restore.py
 src/starrocks_br/schema.py
@@ -40,6 +41,8 @@ tests/test_history.py
 tests/test_labels.py
 tests/test_logger.py
 tests/test_planner.py
+tests/test_prune.py
+tests/test_prune_cli.py
 tests/test_repository_sql.py
 tests/test_restore.py
 tests/test_schema_setup.py

{starrocks_br-0.6.0 → starrocks_br-0.7.0a1}/tests/test_cli_init.py

@@ -53,7 +53,7 @@ def test_init_fails_when_repository_not_found(config_file, mock_db, setup_passwo
         side_effect=RuntimeError(
             "Repository 'test_repo' not found. Please create it first using:\n"
             "  CREATE REPOSITORY test_repo WITH BROKER ON LOCATION '...' PROPERTIES(...)\n"
-            "For examples, see: https://docs.starrocks.io/docs/sql-reference/sql-statements/backup_restore/CREATE_REPOSITORY/"
+            "For examples, see: https://docs.starrocks.io/docs/sql-reference/sql-statements/data-definition/backup_restore/CREATE_REPOSITORY/"
         ),
     )