speedy-utils 1.0.4__py3-none-any.whl → 1.0.5__py3-none-any.whl

speedy_utils/common/clock.py

@@ -0,0 +1,215 @@
+import inspect
+import os
+import time
+
+from loguru import logger
+from tabulate import tabulate
+
+__all__ = ["Clock", "timef"]
+
+
+def timef(func):
+    "Decorator to print the execution time of a function"
+
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        result = func(*args, **kwargs)
+        end_time = time.time()
+        execution_time = end_time - start_time
+        logger.opt(depth=2).info(
+            f"{func.__name__} took {execution_time:0.2f} seconds to execute."
+        )
+        return result
+
+    return wrapper
+
+
+class Clock:
+    """
+    A simple timer utility to measure and log time intervals.
+
+    Usage:
+
+    1. Creating and starting the timer:
+        timer = Timer(start_now=True)
+        # or
+        timer = Timer(start_now=False)
+        timer.start()
+
+    2. Measure time since the timer started:
+        elapsed_time = timer.elapsed_time()
+
+    3. Log the time elapsed since the timer started:
+        timer.log_elapsed_time()
+        # or use a custom logger
+        timer.log_elapsed_time(custom_logger=my_custom_logger)
+
+    4. Measure time since the last checkpoint:
+        time_since_last_checkpoint = timer.time_since_last_checkpoint()
+
+    5. Update a named task in the internal task time table:
+        timer.update_task("task_name")
+
+    6. Print the task time table every 'interval' seconds:
+        timer.print_task_table(interval=1)
+    """
+
+    def __init__(self, start_now=True):
+        """Initialize the timer and optionally start it immediately."""
+        self.start_time = None
+        self.task_times = {}
+        self.last_checkpoint = None
+        if start_now:
+            self.start()
+        self.print_counter = 0
+        self.last_print_time = time.time()
+        self.min_depth = float("inf")
+
+    def start(self):
+        """Start the timer or reset if already started."""
+        if self.start_time is not None:
+            raise ValueError("Timer has already been started.")
+        self.start_time = time.time()
+        self.last_checkpoint = self.start_time
+        # logger.opt(depth=2).info(f"Timer started. {id(self)=}")
+
+    def elapsed_time(self):
+        """Return the time elapsed since the timer started."""
+        if self.start_time is None:
+            raise ValueError("Timer has not been started.")
+        return time.time() - self.start_time
+
+    def log_elapsed_time(self, custom_logger=None):
+        """Log the time elapsed since the timer started."""
+        msg = f"Time elapsed: {self.elapsed_time():.2f} seconds."
+        if custom_logger:
+            custom_logger(msg)
+        else:
+            logger.opt(depth=2).info(msg)
+
+    def _tick(self):
+        """Return the time elapsed since the last checkpoint and update the last checkpoint."""
+        # assert self.start_time is not None, f"Timer has not been started. {id(self)=}"
+        if not self.start_time:
+            logger.opt(depth=2).warning(
+                "Timer has not been started. Please call start() before using this method."
+            )
+            return
+        current_time = time.time()
+        if self.last_checkpoint is None:
+            logger.opt(depth=2).warning(
+                "Last checkpoint is not set. Please call start() before using this method."
+            )
+            return
+        elapsed = current_time - self.last_checkpoint
+        self.last_checkpoint = current_time
+        return elapsed
+
+    def tick(self):
+        return self._tick()
+
+    def time_since_last_checkpoint(self):
+        """Return the time elapsed since the last checkpoint."""
+        if self.start_time is None:
+            # raise ValueError("Timer has not been started.")
+            logger.opt(depth=2).warning(
+                "Timer has not been started. Please call start() before using this method."
+            )
+            return
+        if self.last_checkpoint is None:
+            logger.opt(depth=2).warning(
+                "Last checkpoint is not set. Please call start() before using this method."
+            )
+            return
+        return time.time() - self.last_checkpoint
+
+    def update_task(self, task_name):
+        """Update the elapsed time for the specified task, including file, line, and call depth."""
+
+        # Get the full call stack
+        stack = inspect.stack()
+
+        # Get the file and line number of the caller (the previous frame in the stack)
+        caller_frame = stack[1]
+        file_lineno = f"{os.path.basename(caller_frame.filename)}:{caller_frame.lineno}"
+
+        # Calculate the depth of the current call (i.e., how far it is in the stack)
+        call_depth = (
+            len(stack) - 1
+        )  # Subtract 1 to exclude the current frame from the depth count
+        if call_depth < self.min_depth:
+            self.min_depth = call_depth
+
+        # Update the task time in the internal task table
+        if task_name not in self.task_times:
+            self.task_times[task_name] = {
+                "time": 0,
+                "file_lineno": file_lineno,
+                "depth": call_depth,
+            }
+        self.task_times[task_name]["time"] += self.tick()
+
+    def get_percentage_color(self, percentage):
+        """Return ANSI color code based on percentage."""
+        if percentage >= 75:
+            return "\033[91m"  # Red
+        elif percentage >= 50:
+            return "\033[93m"  # Yellow
+        elif percentage >= 25:
+            return "\033[92m"  # Green
+        else:
+            return "\033[94m"  # Blue
+
+    def print_task_table(self, interval=1, max_depth=None):
+        """Print the task time table at regular intervals."""
+        current_time = time.time()
+
+        if current_time - self.last_print_time > interval:
+            self.print_counter += 1
+            total_time = (
+                sum(data["time"] for data in self.task_times.values()) or 1
+            )  # Avoid division by zero
+
+            # Prepare data for the table
+            table_data = []
+            for task_name, data in self.task_times.items():
+                time_spent = data["time"]
+                file_lineno = data["file_lineno"]
+                depth = data["depth"] - self.min_depth
+                if max_depth is not None and depth > max_depth:
+                    continue
+                percentage = (time_spent / total_time) * 100
+
+                # Get color code based on percentage
+                color_code = self.get_percentage_color(percentage)
+                percentage_str = f"{percentage:.2f} %"
+                colored_percentage = f"{color_code}{percentage_str}\033[0m"
+
+                table_data.append(
+                    [
+                        task_name,
+                        file_lineno,
+                        # depth,
+                        f"{time_spent:.2f} s",
+                        colored_percentage,
+                    ]
+                )
+
+            # Add headers and log using tabulate
+            table = tabulate(
+                table_data,
+                headers=["Task", "File:Line", "Time (s)", "Percentage (%)"],
+                tablefmt="grid",
+            )
+
+            self.last_print_time = current_time
+            # total_time_str = f"\nTotal time elapsed: {total_time:.2f} seconds."
+            logger.opt(depth=2).info(f"\n{table}")
+
+
+# Example of how to instantiate the Timer
+speedy_timer = Clock(start_now=False)
+
+
+# Clock, speedy_timer, timef
+__all__ = ["Clock", "speedy_timer", "timef"]

speedy_utils/common/function_decorator.py

@@ -0,0 +1,66 @@
+import functools
+import time
+import traceback
+from collections.abc import Callable
+from typing import Any, Tuple, Type, Union
+
+from loguru import logger
+
+
+def retry_runtime(
+    sleep_seconds: int = 5,
+    max_retry: int = 12,
+    exceptions: type[Exception] | tuple[type[Exception], ...] = (RuntimeError,),
+) -> Callable:
+    """Decorator that retries the function with exponential backoff on specified runtime exceptions.
+
+    Args:
+        sleep_seconds (int): Initial sleep time between retries in seconds
+        max_retry (int): Maximum number of retry attempts
+        exceptions (Union[Type[Exception], Tuple[Type[Exception], ...]]): Exception types to retry on
+
+    Returns:
+        Callable: Decorated function
+    """
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            last_exception = None
+
+            for attempt in range(1, max_retry + 1):
+                try:
+                    return func(*args, **kwargs)
+
+                except (SyntaxError, NameError, ImportError, TypeError) as e:
+                    # Don't retry on syntax/compilation errors
+                    logger.opt(depth=1).error(
+                        f"Critical error in {func.__name__}: {str(e)}\n{traceback.format_exc()}"
+                    )
+                    raise
+
+                except exceptions as e:
+                    last_exception = e
+                    if attempt == max_retry:
+                        logger.opt(depth=1).error(
+                            f"Function {func.__name__} failed after {max_retry} retries: {str(e)}"
+                        )
+                        raise
+
+                    backoff_time = sleep_seconds * (
+                        2 ** (attempt - 1)
+                    )  # Exponential backoff
+                    logger.opt(depth=1).warning(
+                        f"Attempt {attempt}/{max_retry} failed: {str(e)[:100]}. "
+                        f"Retrying in {backoff_time} seconds."
+                    )
+                    time.sleep(backoff_time)
+
+            return None  # This line should never be reached
+
+        return wrapper
+
+    return decorator
+
+
+__all__ = ["retry_runtime"]

speedy_utils/common/logger.py

@@ -0,0 +1,207 @@
+# utils/utils_print.py
+
+import inspect
+import re
+import sys
+import time
+from collections import OrderedDict
+from typing import Annotated, Literal, Optional
+
+from loguru import logger
+
+
+# A subclass of OrderedDict to automatically evict the oldest item after max_size is exceeded
+class _RateLimitCache(OrderedDict):
+    def __init__(self, max_size: int, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.max_size = max_size
+
+    def __setitem__(self, key, value):
+        # If the key already exists, move it to the end (so it's considered "newer")
+        if key in self:
+            self.move_to_end(key)
+        # Use normal __setitem__
+        super().__setitem__(key, value)
+        # Evict the oldest if we're over capacity
+        if len(self) > self.max_size:
+            self.popitem(last=False)  # pop the *first* item
+
+
+# Create a global rate-limit cache with, say, 2,000 distinct entries max
+_last_log_times = _RateLimitCache(max_size=2000)
+
+
+def setup_logger(
+    level: Annotated[
+        Literal[
+            "Trace",
+            "Debug",
+            "Info",
+            "Success",
+            "Warning",
+            "Error",
+            "Critical",
+            "Disable",
+            "T",
+            "D",
+            "I",
+            "S",
+            "W",
+            "E",
+            "C",
+        ],
+        "The desired log level",
+    ] = "Info",
+    enable_grep: Annotated[str, "Comma-separated patterns for enabling logs"] = "",
+    disable_grep: Annotated[str, "Comma-separated patterns for disabling logs"] = "",
+    min_interval: float = -1,
+    max_cache_entries: int = 2000,
+) -> None:
+    """
+    Setup the logger with a rate-limiting feature:
+    - No more than 1 log from the same file:line within `min_interval` seconds.
+    - Track up to `max_cache_entries` distinct file:line pairs in memory.
+    """
+    # Update the cache size if desired
+    _last_log_times.max_size = max_cache_entries
+
+    # Map the shorthand level to the full name
+    level_mapping = {
+        "T": "TRACE",
+        "D": "DEBUG",
+        "I": "INFO",
+        "S": "SUCCESS",
+        "W": "WARNING",
+        "E": "ERROR",
+        "C": "CRITICAL",
+    }
+    level_str = level_mapping.get(level.upper(), level.upper())
+
+    # Set the log level
+    logger.level(level_str)
+
+    # Remove any existing handlers to avoid duplication
+    logger.remove()
+
+    # Prepare grep patterns
+    enable_patterns = [p.strip() for p in enable_grep.split(",") if p.strip()]
+    disable_patterns = [p.strip() for p in disable_grep.split(",") if p.strip()]
+
+    def log_filter(record):
+        """
+        1. Filters out messages below the specified log level.
+        2. Applies 'enable'/'disable' grep filters.
+        3. Rate-limits same file:line messages if they occur within `min_interval` seconds.
+        4. Enforces a max size on the (file:line) dictionary.
+        """
+        # ---------- 1) Log-level check ----------
+        if record["level"].no < logger.level(level_str).no:
+            return False
+
+        # ---------- 2) Grep pattern handling ----------
+        log_message = f"{record['file']}:{record['line']} ({record['function']})"
+        if enable_patterns and not any(
+            re.search(p, log_message) for p in enable_patterns
+        ):
+            return False
+        if disable_patterns and any(
+            re.search(p, log_message) for p in disable_patterns
+        ):
+            return False
+
+        # ---------- 3) Rate limiting by file:line ----------
+        file_line_key = f"{record['file']}:{record['line']}"
+        now = time.time()
+
+        last_time = _last_log_times.get(file_line_key)
+        if last_time is not None and min_interval > 0:
+            try:
+                if now - last_time < min_interval:
+                    return False  # Skip logging within min_interval
+            except TypeError:
+                # Handle case in tests where last_time might be a mock
+                pass
+
+        # Update the cache with new time (will also handle size eviction)
+        _last_log_times[file_line_key] = now
+        return True
+
+    # Add the handler
+    logger.add(
+        sys.stdout,
+        colorize=True,
+        format=(
+            "<green>{time:HH:mm:ss}</green> | "
+            "<level>{level: <8}</level> | "
+            "<cyan>{file}:{line} ({function})</cyan> - <level>{message}</level>"
+        ),
+        filter=log_filter,
+    )
+
+    # ---------- 4) Handle "DISABLE" level ----------
+    if level_str.upper() == "DISABLE":
+        logger.disable("")
+        logger.info("Logging disabled")
+    else:
+        logger.enable("")
+        logger.debug(f"Logging set to {level_str}")
+
+
+_logged_once_set = set()
+_last_log_intervals = {}
+
+
+def _get_call_site_id(depth=2) -> str:
+    """
+    Generate a unique identifier for the call site based on filename and line number.
+    Adjusts for test environment where frame information may change.
+    """
+    frame = inspect.stack()[depth]
+    # Use a stable identifier in test environment to handle mocking
+    return f"{frame.filename}:{frame.lineno}"
+
+
+def log(
+    msg: str,
+    *,
+    level: Literal["info", "warning", "error", "critical", "success"] = "info",
+    once: bool = False,
+    interval: float | None = None,
+) -> None:
+    """
+    Log a message using loguru with optional `once` and `interval` control.
+
+    Args:
+        msg (str): The log message.
+        level (str): Log level (e.g., "info", "warning").
+        once (bool): If True, log only once per call site.
+        interval (float): If set, log only once every `interval` seconds per call site.
+    """
+    identifier = _get_call_site_id(depth=2)
+
+    # Handle once parameter - check before logging
+    if once and identifier in _logged_once_set:
+        return
+
+    # Handle interval parameter - check before logging
+    if interval is not None:
+        now = time.time()
+        last = _last_log_intervals.get(identifier)
+        if last is not None:
+            try:
+                if now - last < interval:
+                    return
+            except TypeError:
+                # Handle case in tests where last might be a mock
+                pass
+
+    # Log the message
+    fn = getattr(logger.opt(depth=1), level)
+    fn(msg)
+
+    # Update rate-limiting caches after successful logging
+    if once:
+        _logged_once_set.add(identifier)
+
+    if interval is not None:
+        _last_log_intervals[identifier] = time.time()

speedy_utils/common/report_manager.py

@@ -0,0 +1,112 @@
+import os
+from collections import defaultdict
+from datetime import datetime
+
+from fastcore.all import threaded
+
+
+class ReportManager:
+    def __init__(self):
+        self.cache_dir = os.path.expanduser("~/.cache/speedy_utils")
+        os.makedirs(self.cache_dir, exist_ok=True)
+
+    def save_report(self, errors, results, execution_time=None, metadata=None):
+        report_path = os.path.join(
+            self.cache_dir, f"report_{datetime.now().strftime('%m%d_%H%M')}.md"
+        )
+        os.makedirs(os.path.dirname(report_path), exist_ok=True)
+
+        # Group errors by error type
+        error_groups = defaultdict(list)
+        for err in errors[:10]:
+            error_type = err["error"].__class__.__name__
+            error_groups[error_type].append(err)
+
+        md_content = [
+            "# Multi-thread Execution Report",
+            f"\n## Summary (Generated at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')})",
+        ]
+
+        if metadata:
+            md_content.extend(
+                [
+                    "\n### Execution Configuration",
+                    f"- Mode: {metadata['mode']}",
+                    f"- Workers: {metadata['max_workers']}",
+                    f"- Execution type: {metadata['execution_mode']}",
+                    f"- Total inputs: {metadata['total_inputs']}",
+                ]
+            )
+
+        md_content.extend(
+            [
+                f"\n### Results Overview",
+                f"- Total items processed: {len(results)}",
+                f"- Success rate: {(len(results) - len(errors))/len(results)*100:.1f}%",
+                f"- Total errors: {len(errors)}",
+            ]
+        )
+
+        if execution_time:
+            md_content.append(f"- Execution time: {execution_time:.2f}s")
+            md_content.append(
+                f"- Average speed: {len(results)/execution_time:.1f} items/second"
+            )
+
+        if error_groups:
+            md_content.extend(
+                ["\n## Errors by Type", "Click headers to expand error details."]
+            )
+
+            for error_type, errs in error_groups.items():
+                md_content.extend(
+                    [
+                        f"\n<details>",
+                        f"<summary><b>{error_type}</b> ({len(errs)} occurrences)</summary>\n",
+                        "| Index | Input | Error Message |",
+                        "|-------|-------|---------------|",
+                    ]
+                )
+
+                for err in errs:
+                    md_content.append(
+                        f"| {err['index']} | `{err['input']}` | {str(err['error'])} |"
+                    )
+
+                # Add first traceback as example
+                md_content.extend(
+                    [
+                        "\nExample traceback:",
+                        "```python",
+                        errs[0]["traceback"],
+                        "```",
+                        "</details>",
+                    ]
+                )
+
+            # Add a section listing all error indices
+            md_content.extend(
+                [
+                    "\n## Error Indices",
+                    "List of indices for items that encountered errors:",
+                    ", ".join(str(err["index"]) for err in errors),
+                ]
+            )
+
+        md_content.extend(
+            [
+                "\n## Results Summary",
+                f"- Successful executions: {len(results) - len(errors)}",
+                f"- Failed executions: {len(errors)}",
+                "\n<details>",
+                "<summary>First 5 successful results</summary>\n",
+                "```python",
+                str([r for r in results[:5] if r is not None]),
+                "```",
+                "</details>",
+            ]
+        )
+
+        with open(report_path, "w", encoding="utf-8") as f:
+            f.write("\n".join(md_content))
+        print(f"Report saved at: {report_path}")