redis-sizer 0.1.0__py3-none-any.whl

redis_sizer/cli.py

@@ -0,0 +1,345 @@
+import time
+from dataclasses import dataclass
+from enum import Enum
+from typing import Annotated
+
+import typer
+from redis import Redis
+from redis.exceptions import RedisError
+from rich import box
+from rich.console import Console
+from rich.progress import track
+from rich.table import Table
+
+app = typer.Typer()
+
+
+class MemoryUnit(str, Enum):
+    """
+    For displaying memory usage in different units.
+    """
+
+    B = "B"
+    KB = "KB"
+    MB = "MB"
+    GB = "GB"
+
+
+@dataclass
+class TableRow:
+    """
+    Represent a row in the result table.
+    """
+
+    key: str
+    count: str
+    size: str
+    avg_size: str
+    min_size: str
+    max_size: str
+    percentage: str
+
+
+@app.command()
+def analyze(
+    host: str,
+    port: Annotated[int, typer.Option(help="Port number")] = 6379,
+    db: Annotated[int, typer.Option(help="DB number")] = 0,
+    password: Annotated[str | None, typer.Option(help="Password")] = None,
+    socket_timeout: Annotated[int, typer.Option(help="Socket timeout in seconds")] = 10,
+    socket_connect_timeout: Annotated[
+        int, typer.Option(help="Socket connect timeout in seconds")
+    ] = 10,
+    pattern: Annotated[str, typer.Option(help="Pattern to filter keys")] = "*",
+    sample_size: Annotated[int | None, typer.Option(help="Number of keys to sample")] = None,
+    namespace_separator: Annotated[str, typer.Option(help="Separator for key namespaces")] = ":",
+    namespace_level: Annotated[
+        int,
+        typer.Option(
+            min=0, help="Number of namespace levels to aggregate keys by. 0 means no aggregation."
+        ),
+    ] = 0,
+    memory_unit: Annotated[
+        MemoryUnit, typer.Option(help="Memory unit for display in result table")
+    ] = MemoryUnit.B,
+    top: Annotated[
+        int | None, typer.Option(help="Maximum number of rows to display in result table")
+    ] = 10,
+    scan_count: Annotated[int, typer.Option(help="COUNT option for scanning keys")] = 1000,
+    batch_size: Annotated[int, typer.Option(help="Batch size for calculating memory usage")] = 1000,
+):
+    """
+    Analyze memory usage across keys in a Redis database and display the results in a table.
+    """
+    # Start the timer to measure execution time
+    start_time = time.time()
+
+    # Create Console instance for rich output
+    console = Console()
+
+    # Create Redis client
+    redis = Redis(
+        host=host,
+        port=port,
+        db=db,
+        password=password,
+        socket_timeout=socket_timeout,
+        socket_connect_timeout=socket_connect_timeout,
+    )
+
+    # Get the total number of keys in the database
+    try:
+        total_size: int = redis.dbsize()  # type: ignore
+    except RedisError as error:
+        console.print(f"[red]Error occured: {error}[/red]")
+        redis.close()
+        exit(1)
+
+    # If the total size is 0, stop the process
+    if total_size == 0:
+        console.print("[yellow]No keys found in the database.[/yellow]")
+        redis.close()
+        return
+
+    console.print(f"The total number of keys: {total_size}")
+
+    # Scan the keys in the database
+    keys = _scan_keys(
+        redis=redis, pattern=pattern, count=scan_count, sample_size=sample_size, console=console
+    )
+    if not keys:
+        console.print(f"[yellow]No keys found matching the pattern: {pattern}[/yellow]")
+        redis.close()
+        return
+
+    # Get memory usage for each key (in batches)
+    memory_usage_by_key: dict[str, int | None] = {}  # key -> memory usage
+    for i in track(
+        range(0, len(keys), batch_size),
+        description="Calculating memory usage...",
+        console=console,
+    ):
+        batch = keys[i : i + batch_size]
+        try:
+            memory_usage_by_key.update(_get_memory_usage(redis=redis, keys=batch))
+        except RedisError as error:
+            console.print(f"[red]Error occured: {error}[/red]")
+            redis.close()
+            exit(2)
+
+    redis.close()
+
+    # Aggregate memory usage by key groups
+    group_data: dict[str, dict] = {}  # group -> {keys: [], size: 0, sizes: []}
+    overall_min: int | None = None
+    overall_max: int = 0
+    valid_keys_count: int = 0
+    for key in keys:
+        memory_usage: int | None = memory_usage_by_key[key]
+        # Memory usage is None for expired keys
+        if memory_usage is None:
+            continue
+        group: str = _parse_key_group(key, namespace_separator, namespace_level)
+        if group not in group_data:
+            group_data[group] = {"keys": [], "size": 0, "sizes": []}
+        group_data[group]["keys"].append(key)
+        group_data[group]["size"] += memory_usage
+        group_data[group]["sizes"].append(memory_usage)
+        overall_min = memory_usage if overall_min is None else min(overall_min, memory_usage)
+        overall_max = max(overall_max, memory_usage)
+        valid_keys_count += 1
+
+    if valid_keys_count == 0:
+        console.print("[yellow]No valid keys found. The scanned keys might have expired. [/yellow]")
+        return
+
+    # Sort the groups by size in descending order
+    sorted_groups = sorted(
+        (
+            (group, data["size"], data["sizes"], len(data["keys"]))
+            for group, data in group_data.items()
+        ),
+        key=lambda x: x[1],
+        reverse=True,
+    )
+
+    # Get hidden_count to limit the number of displayed groups if top is specified
+    total_key_groups: int = len(sorted_groups)
+    if top is not None and top > 0 and top < total_key_groups:
+        group_data_for_rows = sorted_groups[:top]
+        hidden_count = total_key_groups - top
+    else:
+        group_data_for_rows = sorted_groups
+        hidden_count = 0
+
+    # Prepare the rows
+    rows: list[TableRow] = []
+    factor: int = _get_memory_unit_factor(memory_unit)
+    total_size: int = sum(group["size"] for group in group_data.values())
+    for group, size, sizes, count in group_data_for_rows:
+        display_key: str = group if namespace_level == 0 else f"{group}[dim]*[/dim]"
+        size_converted: float = size / factor
+        display_size: str = (
+            f"{size_converted:.2f}" if memory_unit != MemoryUnit.B else f"{int(size_converted)}"
+        )
+        avg_usage = (sum(sizes) / len(sizes)) / factor if sizes else 0
+        min_usage = min(sizes) / factor if sizes else 0
+        max_usage = max(sizes) / factor if sizes else 0
+        percentage = (size / total_size * 100) if total_size else 0
+        rows.append(
+            TableRow(
+                key=display_key,
+                count=str(count),
+                size=display_size,
+                avg_size=f"{avg_usage:.4f}" if memory_unit != MemoryUnit.B else f"{int(avg_usage)}",
+                min_size=f"{min_usage:.4f}" if memory_unit != MemoryUnit.B else f"{int(min_usage)}",
+                max_size=f"{max_usage:.4f}" if memory_unit != MemoryUnit.B else f"{int(max_usage)}",
+                percentage=f"{percentage:.2f}",
+            )
+        )
+
+    # Prepare the total row
+    total_usage_display: str = (
+        f"{total_size / factor:.2f}"
+        if memory_unit != MemoryUnit.B
+        else f"{int(total_size / factor)}"
+    )
+    overall_avg: float = (total_size / valid_keys_count) / factor
+    overall_min_conv: float = (overall_min or 0) / factor
+    overall_max_conv: float = overall_max / factor
+    overall_avg_display: str = (
+        f"{overall_avg:.4f}" if memory_unit != MemoryUnit.B else f"{int(overall_avg)}"
+    )
+    overall_min_display: str = (
+        f"{overall_min_conv:.4f}" if memory_unit != MemoryUnit.B else f"{int(overall_min_conv)}"
+    )
+    overall_max_display: str = (
+        f"{overall_max_conv:.4f}" if memory_unit != MemoryUnit.B else f"{int(overall_max_conv)}"
+    )
+    total_count: int = sum(len(group["keys"]) for group in group_data.values())
+    total_row: TableRow = TableRow(
+        key="Total Keys Scanned",
+        count=str(total_count),
+        size=total_usage_display,
+        avg_size=overall_avg_display,
+        min_size=overall_min_display,
+        max_size=overall_max_display,
+        percentage="100.00",
+    )
+
+    _print_memory_usage_table(
+        rows=rows,
+        total_row=total_row,
+        hidden_count=hidden_count,
+        memory_unit=memory_unit,
+        console=console,
+    )
+
+    # Print the total execution time taken
+    console.print(f"Took {(time.time() - start_time):.2f} seconds")
+
+
+def _scan_keys(
+    redis: Redis, pattern: str, count: int, sample_size: int | None, console: Console
+) -> list[str]:
+    """
+    Scan keys in the Redis database using the given pattern.
+    If sample_size is None, perform a full iteration.
+    """
+    keys = []
+    for key in track(
+        redis.scan_iter(match=pattern, count=count),
+        description="Scanning keys...",
+        total=sample_size,
+        console=console,
+        show_speed=False,
+    ):
+        keys.append(key.decode())
+        if sample_size and sample_size <= len(keys):
+            break
+    return keys
+
+
+def _parse_key_group(key: str, separator: str, level: int) -> str:
+    """
+    Parse the key group of a key up to the specified level using the given separator.
+    For example, _parse_key_group("a:b:c:d", ":", 2) -> "a:b"
+
+    If level is 0, return the entire key.
+    """
+    if level == 0:
+        return key
+    return separator.join(key.split(separator)[:level])
+
+
+def _get_memory_usage(redis: Redis, keys: list[str]) -> dict[str, int | None]:
+    """
+    Get memory usage for a list of keys using Lua script.
+    If the key doesn't exist (expired), memory usage is None.
+    """
+    script = """
+    local result = {}
+    for i, key in ipairs(KEYS) do
+        result[i] = redis.call('MEMORY', 'USAGE', key, 'SAMPLES', 0)
+    end
+    return result
+    """
+    func = redis.register_script(script)
+    return dict(zip(keys, func(keys=keys)))  # type: ignore
+
+
+def _get_memory_unit_factor(memory_unit: MemoryUnit) -> int:
+    """
+    Get the conversion factor for the specified memory unit.
+    """
+    if memory_unit == MemoryUnit.B:
+        return 1
+    elif memory_unit == MemoryUnit.KB:
+        return 1024
+    elif memory_unit == MemoryUnit.MB:
+        return 1024 * 1024
+    elif memory_unit == MemoryUnit.GB:
+        return 1024 * 1024 * 1024
+    else:
+        raise ValueError("Invalid value for memory_unit. Use B, KB, MB, or GB.")
+
+
+def _print_memory_usage_table(
+    rows: list[TableRow],
+    total_row: TableRow,
+    hidden_count: int,
+    memory_unit: MemoryUnit,
+    console: Console,
+):
+    table: Table = Table(box=box.MINIMAL)
+    table.add_column("Key", justify="left")
+    table.add_column("Count", justify="right", style="green")
+    table.add_column(f"Size ({memory_unit.upper()})", justify="right", style="magenta")
+    table.add_column(f"Avg Size ({memory_unit.upper()})", justify="right", style="orange1")
+    table.add_column(f"Min Size ({memory_unit.upper()})", justify="right", style="yellow")
+    table.add_column(f"Max Size ({memory_unit.upper()})", justify="right", style="red")
+    table.add_column("Memory Usage (%)", justify="right", style="cyan")
+    for row in rows:
+        table.add_row(
+            row.key, row.count, row.size, row.avg_size, row.min_size, row.max_size, row.percentage
+        )
+    if 0 < hidden_count:
+        table.add_row(
+            f"... {hidden_count} more entries not shown ...", "", "", "", "", "", "", style="dim"
+        )
+    table.add_section()
+    table.add_row(
+        total_row.key,
+        total_row.count,
+        total_row.size,
+        total_row.avg_size,
+        total_row.min_size,
+        total_row.max_size,
+        total_row.percentage,
+        style="bold",
+    )
+    console.print(table)
+
+
+if __name__ == "__main__":
+    app()

redis_sizer-0.1.0.dist-info/METADATA

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: redis-sizer
+Version: 0.1.0
+Summary: A simple command-line tool for analyzing memory usage across keys in a Redis database.
+Author-Email: Mienxiu <mienxiu@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Project-URL: Homepage, https://github.com/mienxiu/redis-sizer
+Project-URL: Documentation, https://github.com/mienxiu/redis-sizer
+Project-URL: Repository, https://github.com/mienxiu/redis-sizer
+Requires-Python: >=3.11
+Requires-Dist: typer<0.16.0,>=0.15.2
+Requires-Dist: redis<6.0.0,>=5.2.1
+Description-Content-Type: text/markdown
+
+# redis-sizer
+
+A simple command-line tool for analyzing memory usage across keys in a Redis database.
+
+
+![Demo](docs/demo.webp)
+
+redis-sizer can help with the following problems:
+- Optimize caching strategy.
+- Gain insights into memory usage at different namespace levels.
+- Identify which parts of your application consume the most memory.
+
+## Installation
+
+Install via pip:
+
+```bash
+pip install redis-sizer
+```
+
+## Usage
+
+Run redis-sizer by specifying the Redis host along with any desired options:
+
+```bash
+redis-sizer [OPTIONS] HOST
+```
+
+Options:
+
+- `--port`: Port number [default: 6379]
+- `--db`: DB number [default: 0]
+- `--password`: Password [default: None]
+- `--socket-timeout`: Socket timeout in seconds [default: 10]
+- `--socket-connect-timeout`: Socket connect timeout in seconds [default: 10]
+- `--pattern`: Pattern to filter keys [default: *]
+- `--sample-size`: Number of keys to sample [default: None]
+- `--namespace-separator`: Separator for key namespaces [default: :]
+- `--namespace-level`: Number of namespace levels to aggregate keys by. 0 means no aggregation. [default: 0]
+- `--memory-unit`: Memory unit for display in result table [default: B]
+- `--top`: Maximum number of rows to display in result table [default: 10]
+- `--scan-count`: `COUNT` option for scanning keys [default: 1000]
+- `--batch-size`: Batch size for calculating memory usage [default: 1000]
+
+### Aggregating Memory Usage by Key Groups
+
+Many applications using Redis adopt hierarchical key naming conventions.
+redis-sizer leverages this structure to aggregate and analyze memory usage, helping you quickly identify which key groups consume the most memory.
+
+For example, if your keys follow a colon-delimited format (e.g., `users:profiles:001`, `users:logs:login:002`), the hierarchy can be visualized like a tree structure:
+
+```
+DB
+├── users:
+│   ├── profiles:
+│   │   ├── 001
+│   │   └── 002
+│   └── logs:
+│       ├── 001
+│       └── 002
+├── sessions:
+│   └── active:
+...
+```
+
+Use `--namespace-level 1` to aggregate memory usage at the top-level namespace:
+
+```console
+  Key                │  Count  │ Size (MB) │ ... │ Memory Usage (%)  
+╶────────────────────┼─────────┼───────────┼─────┼──────────────────╴
+  users              │ 350000  │   2460.86 │ ... │           93.82  
+  sessions           │  60000  │    140.23 │ ... │            5.35  
+  cache              │  10000  │     20.07 │ ... │            0.77  
+  tokens             │    700  │      1.75 │ ... │            0.07  
+╶────────────────────┼─────────┼───────────┼─────┼──────────────────╴
+  Total Keys Scanned │ 420700  │   2622.90 │ ... │          100.00  
+```
+
+Use `--namespace-level 2` to aggregate memory usage at the second-level namespace:
+
+```console
+  Key                │  Count  │ Size (MB) │ ... │ Memory Usage (%)  
+╶────────────────────┼─────────┼───────────┼─────┼──────────────────╴
+  users:profiles     │ 100000  │   1177.22 │ ... │            44.88  
+  users:logs         │ 200000  │    988.01 │ ... │            37.67  
+  sessions:active    │  50000  │    295.64 │ ... │            11.27  
+  cache:images       │  40000  │    100.02 │ ... │             3.81  
+  cache:queries      │  20000  │     40.21 │ ... │             1.53  
+  tokens:auth        │    700  │      1.75 │ ... │             0.07  
+╶────────────────────┼─────────┼───────────┼─────┼──────────────────╴
+  Total Keys Scanned │ 420700  │   2622.90 │ ... |           100.00  
+```
+
+## Considerations
+
+- Since data can change during analysis, the final result may not accurately reflect the database’s state at the beginning of the process.
+- For databases with large collections of keys, consider using the `--sample-size` option to balance processing time with accuracy. If `--sample-size` is not specified, redis-sizer will perform perform a full iteration.
+- Increasing `--scan-count` or `--batch-size` can reduce network overhead. However, this can increase the load on the Redis server without providing meaningful performance improvements and, in some cases, may even lead to server blocking. Adjust these values carefully based on your server’s capacity.
+
+## Redis Version Compatibility
+
+- This tool relies on the Redis [`MEMORY USAGE`](https://redis.io/docs/latest/commands/memory-usage/) command, which has been available since version 4.0.0.
+- It has been tested and verified to work with Redis versions 4, 5, 6, and 7.
+
+## Dependencies
+
+redis-sizer depends on:
+
+- [redis-py](https://github.com/redis/redis-py): to communicate with the Redis database.
+- [typer](https://github.com/fastapi/typer): to build CLI application, with output formatting supported by [rich](https://github.com/Textualize/rich)

redis_sizer-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+redis_sizer-0.1.0.dist-info/METADATA,sha256=1YVz7J73PCvoZ4wuXjwZKL4i5-u7KeqsHliVHKuYCLM,5959
+redis_sizer-0.1.0.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+redis_sizer-0.1.0.dist-info/entry_points.txt,sha256=t5Sc1RaT98-2dbgQPtD_Ti7OS6xWkoli3UFMhYAY1IQ,68
+redis_sizer-0.1.0.dist-info/licenses/LICENSE,sha256=8293ucsd6ba-E6wZGqPN9rUhSFJF50AHfBuvr7rVGYA,1064
+redis_sizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+redis_sizer/cli.py,sha256=FB1whUBWXrUt4udouXEcZI9EhGfVIjN5MHEN0ARCOrs,11585
+redis_sizer-0.1.0.dist-info/RECORD,,

redis_sizer-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: pdm-backend (2.4.3)
+Root-Is-Purelib: true
+Tag: py3-none-any

redis_sizer-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+redis-sizer = redis_sizer.cli:app
+
+[gui_scripts]
+

redis_sizer-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Mienxiu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.