starrocks-br 0.2.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

starrocks_br/schema.py

@@ -1,32 +1,33 @@
 from . import logger
 
+
 def initialize_ops_schema(db) -> None:
     """Initialize the ops database and all required control tables.
-    
+
     Creates empty ops tables. Does NOT populate with sample data.
     Users must manually insert their table inventory records.
     """
-    
+
     logger.info("Creating ops database...")
     db.execute("CREATE DATABASE IF NOT EXISTS ops")
     logger.success("ops database created")
-    
+
     logger.info("Creating ops.table_inventory...")
     db.execute(get_table_inventory_schema())
     logger.success("ops.table_inventory created")
-    
+
     logger.info("Creating ops.backup_history...")
     db.execute(get_backup_history_schema())
     logger.success("ops.backup_history created")
-    
+
     logger.info("Creating ops.restore_history...")
     db.execute(get_restore_history_schema())
     logger.success("ops.restore_history created")
-    
+
     logger.info("Creating ops.run_status...")
     db.execute(get_run_status_schema())
     logger.success("ops.run_status created")
-    
+
     logger.info("Creating ops.backup_partitions...")
     db.execute(get_backup_partitions_schema())
     logger.success("ops.backup_partitions created")
@@ -36,26 +37,26 @@ def initialize_ops_schema(db) -> None:
 
 def ensure_ops_schema(db) -> bool:
     """Ensure ops schema exists, creating it if necessary.
-    
+
     Returns True if schema was created, False if it already existed.
     This is called automatically before backup/restore operations.
     """
     try:
         result = db.query("SHOW DATABASES LIKE 'ops'")
-        
+
         if not result:
             initialize_ops_schema(db)
             return True
-        
+
         db.execute("USE ops")
         tables_result = db.query("SHOW TABLES")
-        
+
         if not tables_result or len(tables_result) < 5:
             initialize_ops_schema(db)
             return True
-        
+
         return False
-        
+
     except Exception:
         initialize_ops_schema(db)
         return True
@@ -71,8 +72,9 @@ def get_table_inventory_schema() -> str:
         created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
         updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
     )
-    PRIMARY KEY (inventory_group, database_name, table_name)
+    UNIQUE KEY (inventory_group, database_name, table_name)
     COMMENT "Inventory groups mapping to databases/tables (supports '*' wildcard)"
+    DISTRIBUTED BY HASH(inventory_group)
     """
 
 
@@ -133,12 +135,14 @@ def get_backup_partitions_schema() -> str:
     """Get CREATE TABLE statement for backup_partitions."""
     return """
     CREATE TABLE IF NOT EXISTS ops.backup_partitions (
+        key_hash STRING NOT NULL COMMENT "MD5 hash of composite key (label, database_name, table_name, partition_name)",
         label STRING NOT NULL COMMENT "The backup label this partition belongs to. FK to ops.backup_history.label.",
         database_name STRING NOT NULL COMMENT "The name of the database the partition belongs to.",
         table_name STRING NOT NULL COMMENT "The name of the table the partition belongs to.",
         partition_name STRING NOT NULL COMMENT "The name of the specific partition.",
         created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT "Timestamp when this record was created."
     )
-    PRIMARY KEY (label, database_name, table_name, partition_name)
+    PRIMARY KEY (key_hash)
     COMMENT "Tracks every partition included in a backup snapshot."
-    """
+    DISTRIBUTED BY HASH(key_hash)
+    """

starrocks_br/timezone.py

@@ -5,10 +5,10 @@ from zoneinfo import ZoneInfo
 
 def get_current_time_in_cluster_tz(cluster_tz: str) -> str:
     """Get current time formatted in cluster timezone.
-    
+
     Args:
         cluster_tz: Timezone string (e.g., 'Asia/Shanghai', 'UTC', '+08:00')
-        
+
     Returns:
         Formatted datetime string in 'YYYY-MM-DD HH:MM:SS' format in the cluster timezone
     """
@@ -19,58 +19,58 @@ def get_current_time_in_cluster_tz(cluster_tz: str) -> str:
 
 def parse_datetime_with_tz(dt_str: str, tz: str) -> datetime.datetime:
     """Parse datetime string assuming the given timezone.
-    
+
     Args:
         dt_str: Datetime string in 'YYYY-MM-DD HH:MM:SS' format
         tz: Timezone string (e.g., 'Asia/Shanghai', 'UTC', '+08:00')
-        
+
     Returns:
         Timezone-aware datetime object
     """
     timezone = _get_timezone(tz)
-    
+
     dt = datetime.datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
     dt = dt.replace(tzinfo=timezone)
-    
+
     return dt
 
 
 def normalize_datetime_to_tz(dt: datetime.datetime, target_tz: str) -> datetime.datetime:
     """Convert datetime to target timezone.
-    
+
     Args:
         dt: Datetime object (timezone-aware or naive)
         target_tz: Target timezone string (e.g., 'Asia/Shanghai', 'UTC', '+08:00')
-        
+
     Returns:
         Timezone-aware datetime object in the target timezone
     """
     timezone = _get_timezone(target_tz)
-    
+
     if dt.tzinfo is None:
         dt = dt.replace(tzinfo=datetime.timezone.utc)
-    
+
     dt = dt.astimezone(timezone)
-    
+
     return dt
 
 
 def _get_timezone(tz_str: str) -> Union[ZoneInfo, datetime.timezone]:
     """Get timezone object from timezone string.
-    
+
     Handles both named timezones (e.g., 'Asia/Shanghai') and offset strings (e.g., '+08:00', '-05:00').
-    
+
     Args:
         tz_str: Timezone string
-        
+
     Returns:
         ZoneInfo or timezone object
     """
     tz_str = tz_str.strip()
-    
+
     if tz_str.upper() == "UTC" or tz_str == "+00:00" or tz_str == "-00:00":
         return ZoneInfo("UTC")
-    
+
     if tz_str.startswith(("+", "-")):
         try:
             hours, minutes = _parse_offset(tz_str)
@@ -78,7 +78,7 @@ def _get_timezone(tz_str: str) -> Union[ZoneInfo, datetime.timezone]:
             return datetime.timezone(offset)
         except ValueError:
             return ZoneInfo("UTC")
-    
+
     try:
         return ZoneInfo(tz_str)
     except Exception:
@@ -87,13 +87,13 @@ def _get_timezone(tz_str: str) -> Union[ZoneInfo, datetime.timezone]:
 
 def _parse_offset(offset_str: str) -> tuple[int, int]:
     """Parse timezone offset string to hours and minutes.
-    
+
     Args:
         offset_str: Offset string in format '+HH:MM' or '-HH:MM'
-        
+
     Returns:
         Tuple of (hours, minutes)
-        
+
     Raises:
         ValueError: If offset string is invalid, including:
             - String length < 6 characters
@@ -103,23 +103,22 @@ def _parse_offset(offset_str: str) -> tuple[int, int]:
     """
     if len(offset_str) < 6:
         raise ValueError(f"Invalid offset format: {offset_str}")
-    
-    if offset_str[3] != ':':
+
+    if offset_str[3] != ":":
         raise ValueError(f"Invalid offset format: {offset_str} (missing colon)")
-    
-    sign = 1 if offset_str[0] == '+' else -1
-    
+
+    sign = 1 if offset_str[0] == "+" else -1
+
     try:
         hours = int(offset_str[1:3])
         minutes = int(offset_str[4:6])
     except ValueError as e:
         raise ValueError(f"Invalid offset format: {offset_str} (non-numeric values)") from e
-    
+
     if hours < 0 or hours >= 24:
         raise ValueError(f"Invalid offset format: {offset_str} (hours must be 00-23)")
-    
+
     if minutes < 0 or minutes >= 60:
         raise ValueError(f"Invalid offset format: {offset_str} (minutes must be 00-59)")
-    
-    return sign * hours, sign * minutes
 
+    return sign * hours, sign * minutes

starrocks_br/utils.py

@@ -0,0 +1,86 @@
+def quote_identifier(identifier):
+    """
+    Quote a SQL identifier (database, table, or column name) with backticks.
+
+    Args:
+        identifier: The database, table, or column name to quote
+
+    Returns:
+        The identifier wrapped in backticks with internal backticks escaped
+
+    Raises:
+        ValueError: If identifier is None or empty string
+
+    Examples:
+        >>> quote_identifier("my_table")
+        '`my_table`'
+        >>> quote_identifier("select")
+        '`select`'
+        >>> quote_identifier("table`with`ticks")
+        '`table``with``ticks`'
+    """
+    if identifier is None:
+        raise ValueError("Identifier cannot be None")
+
+    if identifier == "":
+        raise ValueError("Identifier cannot be empty")
+
+    escaped = identifier.replace("`", "``")
+    return f"`{escaped}`"
+
+
+def quote_value(value):
+    """
+    Quote and escape a SQL string value for safe query interpolation.
+
+    Args:
+        value: The string value to quote and escape
+
+    Returns:
+        The properly quoted and escaped SQL value
+
+    Examples:
+        >>> quote_value("test")
+        "'test'"
+        >>> quote_value("O'Brien")
+        "'O''Brien'"
+        >>> quote_value(None)
+        'NULL'
+    """
+    if value is None:
+        return "NULL"
+
+    value = str(value)
+    escaped = value.replace("\\", "\\\\")
+    escaped = escaped.replace("'", "''")
+    escaped = escaped.replace("\n", "\\n")
+    escaped = escaped.replace("\t", "\\t")
+
+    return f"'{escaped}'"
+
+
+def build_qualified_table_name(database, table):
+    """
+    Build a fully qualified table name with proper quoting.
+
+    Args:
+        database: The database name
+        table: The table name
+
+    Returns:
+        Fully qualified table name in format `database`.`table`
+
+    Raises:
+        ValueError: If database or table is None or empty
+
+    Examples:
+        >>> build_qualified_table_name("my_db", "my_table")
+        '`my_db`.`my_table`'
+    """
+    if not database:
+        raise ValueError("Database name cannot be empty or None")
+
+    if not table:
+        raise ValueError("Table name cannot be empty or None")
+
+    return f"{quote_identifier(database)}.{quote_identifier(table)}"

starrocks_br-0.4.0.dist-info/METADATA

@@ -0,0 +1,152 @@
+Metadata-Version: 2.4
+Name: starrocks-br
+Version: 0.4.0
+Summary: StarRocks Backup and Restore automation tool
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: click<9,>=8.1.7
+Requires-Dist: PyYAML<7,>=6.0.1
+Requires-Dist: mysql-connector-python<10,>=9.0.0
+Provides-Extra: dev
+Requires-Dist: pytest<9,>=8.3.2; extra == "dev"
+Requires-Dist: pytest-mock<4,>=3.14.0; extra == "dev"
+Requires-Dist: pytest-cov<6,>=5.0.0; extra == "dev"
+Requires-Dist: ruff<1,>=0.8.0; extra == "dev"
+
+# StarRocks Backup & Restore
+
+Full and incremental backup automation for StarRocks shared-nothing clusters.
+
+**Requirements:** StarRocks 3.5+ (shared-nothing mode)
+
+📋 **[Release Notes & Changelog](CHANGELOG.md)**
+
+## Table of Contents
+
+- [Why This Tool?](#why-this-tool)
+- [Documentation](#documentation)
+- [Installation](#installation)
+- [Configuration](#configuration)
+- [Basic Usage](#basic-usage)
+- [How It Works](#how-it-works)
+
+## Why This Tool?
+
+StarRocks provides native `BACKUP` and `RESTORE` commands, but they only support full backups. For large-scale deployments hosting data at petabyte scale, full backups are not feasible due to time, storage, and network constraints.
+
+This tool adds **incremental backup capabilities** to StarRocks by leveraging native partition-based backup features.
+
+**What StarRocks doesn't provide:**
+- ❌ **No incremental backups** - You must manually identify changed partitions and build complex backup commands
+- ❌ **No backup history** - No built-in way to track what was backed up, when, or which backups succeeded/failed
+- ❌ **No restore intelligence** - You manually determine which backups are needed for point-in-time recovery
+- ❌ **No organization** - No way to group tables or manage different backup strategies
+- ❌ **No concurrency control** - Multiple backup operations can conflict
+
+**What this tool provides:**
+- ✅ **Automatic incremental backups** - Tool detects changed partitions since the last full backup automatically
+- ✅ **Complete operation tracking** - Every backup and restore is logged with status, timestamps, and error details
+- ✅ **Intelligent restore** - Automatically resolves backup chains (full + incremental) for you
+- ✅ **Inventory groups** - Organize tables into groups with different backup strategies
+- ✅ **Job concurrency control** - Prevents conflicting operations
+- ✅ **Safe restores** - Atomic rename mechanism prevents data loss during restore
+- ✅ **Metadata management** - Dedicated `ops` database tracks all backup metadata and partition manifests
+
+In short: this tool transforms StarRocks's basic backup/restore commands into a **production-ready incremental backup solution**.
+
+## Documentation
+
+- **[Getting Started](docs/getting-started.md)** - Step-by-step tutorial
+- **[Core Concepts](docs/core-concepts.md)** - Understand inventory groups, backup types, and restore chains
+- **[Installation](docs/installation.md)** - All installation methods
+- **[Configuration](docs/configuration.md)** - Config file reference and TLS setup
+- **[Commands](docs/commands.md)** - Detailed command reference
+- **[Scheduling & Monitoring](docs/scheduling.md)** - Automate backups and monitor status
+
+## Installation
+
+### Option 1: PyPI
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip install starrocks-br
+```
+
+### Option 2: Standalone Executable
+
+Download from [releases](https://github.com/deep-bi/starrocks-br/releases/latest):
+
+```bash
+# Linux
+chmod +x starrocks-br-linux-x86_64
+mv starrocks-br-linux-x86_64 starrocks-br
+./starrocks-br --help
+```
+
+See [Installation Guide](docs/installation.md) for all options.
+
+## Configuration
+
+Create a `config.yaml` file pointing to your StarRocks cluster:
+
+```yaml
+host: "127.0.0.1"       # StarRocks FE node address
+port: 9030              # MySQL protocol port
+user: "root"            # Database user with backup/restore privileges
+database: "your_database"   # Database containing tables to backup
+repository: "your_repo_name"  # Repository created via CREATE REPOSITORY in StarRocks
+```
+
+Set password:
+```bash
+export STARROCKS_PASSWORD="your_password"
+```
+
+See [Configuration Reference](docs/configuration.md) for TLS and advanced options.
+
+## Basic Usage
+
+**Initialize:**
+```bash
+starrocks-br init --config config.yaml
+```
+
+**Define inventory groups** (in StarRocks):
+```sql
+INSERT INTO ops.table_inventory (inventory_group, database_name, table_name)
+VALUES
+  ('production', 'mydb', 'users'),
+  ('production', 'mydb', 'orders');
+```
+
+**Backup:**
+```bash
+# Full backup
+starrocks-br backup full --config config.yaml --group production
+
+# Incremental backup (tool detects changed partitions automatically)
+starrocks-br backup incremental --config config.yaml --group production
+```
+
+**Restore:**
+```bash
+# Tool automatically resolves backup chains
+starrocks-br restore --config config.yaml --target-label mydb_20251118_full
+```
+
+See [Commands Reference](docs/commands.md) for all options.
+
+## How It Works
+
+1. **Inventory Groups**: Define collections of tables that share the same backup strategy
+2. **ops Database**: Tool creates an `ops` database to track all operations and metadata
+3. **Automatic Incrementals**: Tool queries partition metadata and compares with the baseline to detect changes
+4. **Intelligent Restore**: Automatically resolves backup chains (full + incremental) for point-in-time recovery
+5. **Safe Operations**: All restores use temporary tables with atomic rename for safety
+
+Read [Core Concepts](docs/core-concepts.md) for detailed explanations.
+
+## Contributing
+
+We welcome contributions! See issues for areas that need help or create a new issue to report a bug or request a feature.

starrocks_br-0.4.0.dist-info/RECORD

@@ -0,0 +1,21 @@
+starrocks_br/__init__.py,sha256=i1m0FIl2IAXaVyNoya0ZNAx3WfhIp9I6VLhTz06qNFY,28
+starrocks_br/cli.py,sha256=yXrmfLL-doIZ9ahFKfXqGR09NB08jqz4BUhkkSCpEJc,20483
+starrocks_br/concurrency.py,sha256=N0LD4VHTAFNhD4YslrkOCDSx5cnR5rCEkNH9MkODxv8,5903
+starrocks_br/config.py,sha256=APqOZcJuUzYmGNHoJRlsu4l3sWl_4SS1kRLKjKm2Oag,2059
+starrocks_br/db.py,sha256=gPFP_VcGmAM5uMADSFR9pNEbYN9IXDnN0O2YFMptkmM,4962
+starrocks_br/executor.py,sha256=Z8k5oW-TRE0zg8on9_YtKqhT82Nq_ZmuGxC09Wx7M60,11367
+starrocks_br/health.py,sha256=rmkgNYf6kk3VDZx-PmnAG3lzmtvnJcUPG7Ppb6BA7IU,1021
+starrocks_br/history.py,sha256=kx2H0wGVYCoIvC1_zN4BMGv9d2YebR5oX3NlUd9yeWg,3005
+starrocks_br/labels.py,sha256=-JfO34gNjYEGjSRW5haRRYYWvMrXRpNh9mTKXthAxzI,1675
+starrocks_br/logger.py,sha256=QTfr-nC3TdeU7f1gcRTRDAQSLYpwaevd_iT1B_RbuF8,900
+starrocks_br/planner.py,sha256=QcgBw_psqzWS2FnsmXvYHFgiOMMkfyEvMJryDUFhkpk,10926
+starrocks_br/repository.py,sha256=gZgT0mAjs-AAdESXPF8Syv0bE8m5njya5leTageElQ8,1251
+starrocks_br/restore.py,sha256=4XBcPMeplrqKyaU_yU5GhunAtPWtmvErOi2xOQPWQKA,20306
+starrocks_br/schema.py,sha256=FSJjcz4q3SU_rHLptsSzrlm-o0dcvIu6LbpT-Z5GyZA,6199
+starrocks_br/timezone.py,sha256=RYmM9RIEy1hphVGhwv7A8j1DyV9Djf-Y5HE8NxZMDtg,3609
+starrocks_br/utils.py,sha256=LF3uBdaNMeslE4UHl_wwv4QErCS48ISxpPqYX8dbrc8,2176
+starrocks_br-0.4.0.dist-info/METADATA,sha256=wYmVX_4K_i5K42fxA075GEUqCBSAbhfbAJ-qnvPx8I8,5675
+starrocks_br-0.4.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+starrocks_br-0.4.0.dist-info/entry_points.txt,sha256=AKUt01G2MAlh85s1Q9kNQDOUio14kaTnT3dmg9gjdNg,54
+starrocks_br-0.4.0.dist-info/top_level.txt,sha256=CU1tGVo0kjulhDr761Sndg-oTeRKsisDnWm8UG95aBE,13
+starrocks_br-0.4.0.dist-info/RECORD,,

starrocks_br-0.2.0.dist-info/METADATA

@@ -1,12 +0,0 @@
-Metadata-Version: 2.4
-Name: starrocks-br
-Version: 0.2.0
-Summary: StarRocks Backup and Restore automation tool
-Requires-Python: >=3.9
-Requires-Dist: click<9,>=8.1.7
-Requires-Dist: PyYAML<7,>=6.0.1
-Requires-Dist: mysql-connector-python<10,>=9.0.0
-Provides-Extra: dev
-Requires-Dist: pytest<9,>=8.3.2; extra == "dev"
-Requires-Dist: pytest-mock<4,>=3.14.0; extra == "dev"
-Requires-Dist: pytest-cov<6,>=5.0.0; extra == "dev"

starrocks_br-0.2.0.dist-info/RECORD

@@ -1,20 +0,0 @@
-starrocks_br/__init__.py,sha256=i1m0FIl2IAXaVyNoya0ZNAx3WfhIp9I6VLhTz06qNFY,28
-starrocks_br/cli.py,sha256=3FvtMDYra6_9CwxSWj3i4VcIjHlSDO9LbBwC0-Bh8bc,20550
-starrocks_br/concurrency.py,sha256=wx69u-RW1OnukKn6CQ9EJS4L42N9Gzw7Xz7rgETzmy4,5934
-starrocks_br/config.py,sha256=nL57JR4O1WBG6iOscuCX5aui7lOs6PIiS97MIgVDvOM,2036
-starrocks_br/db.py,sha256=UDmN8uIYTMzxx7_D0XxmYSpH3VTwBKEeBNQ-4hI4SnA,5098
-starrocks_br/executor.py,sha256=I9cefPbHaSiajPvoYBVaRXM06UfP6jzVRdS8q_t5zeE,10681
-starrocks_br/health.py,sha256=DpTy4uqk1UrbV0d9Wtk9Ke9K0iT4ndL-01gqqSywR_c,1050
-starrocks_br/history.py,sha256=j6eqkD1MyTvgoztffnLnr6-6VXd0gdvLxLLKxbC1AG0,3016
-starrocks_br/labels.py,sha256=D67JqIUWtFAnuj9thnC4Y7A0tzLk6d4YpBtDGhen1yc,1689
-starrocks_br/logger.py,sha256=QTfr-nC3TdeU7f1gcRTRDAQSLYpwaevd_iT1B_RbuF8,900
-starrocks_br/planner.py,sha256=_e65v5XRoXKJcLPX73UXKtA721TWjw2Txfet-TItXu8,10326
-starrocks_br/repository.py,sha256=6uTJBYgQFEjJBlfhirriTkad80teTPcBSl_tphUExz4,1269
-starrocks_br/restore.py,sha256=_OzlQjPFkH7ySgFSSbpYZHY3A_U1Jz3-UKTwGl2AiQw,18965
-starrocks_br/schema.py,sha256=s_BAUhNgfQscRzhj-OB4xmp19CYZ0ZElyBfdht0_OSE,6114
-starrocks_br/timezone.py,sha256=ONaudOgIfzPdJpZyIUSh0HF5D7oPQAtungNoUqtGM6M,3738
-starrocks_br-0.2.0.dist-info/METADATA,sha256=RnBcIZXdCArx1rXruABPf735XJjb3ZB1jy-egUrKBy0,419
-starrocks_br-0.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-starrocks_br-0.2.0.dist-info/entry_points.txt,sha256=AKUt01G2MAlh85s1Q9kNQDOUio14kaTnT3dmg9gjdNg,54
-starrocks_br-0.2.0.dist-info/top_level.txt,sha256=CU1tGVo0kjulhDr761Sndg-oTeRKsisDnWm8UG95aBE,13
-starrocks_br-0.2.0.dist-info/RECORD,,