cloudmesh-ai-common 7.0.4__py3-none-any.whl

cloudmesh/ai/common/__init__.py

@@ -0,0 +1,48 @@
+"""Common utilities for cloudmesh-ai components.
+
+This package provides shared functionality for logging, telemetry, system 
+information gathering, and other common helper utilities used across 
+the cloudmesh-ai ecosystem.
+"""
+
+from .logging import (
+    set_context_id,
+    get_context_id,
+    ContextFilter,
+    JsonFormatter,
+    load_logging_config,
+    get_log_dir,
+    ensure_log_dir,
+    get_log_file_path,
+    get_logger,
+    progress,
+)
+from .aggregation import TelemetryAggregator
+from .telemetry import (
+    TelemetryBackend,
+    JSONFileBackend,
+    SQLiteBackend,
+    TextBackend,
+    Telemetry,
+    AsyncTelemetry,
+)
+from .sys import (
+    os_is_windows,
+    os_is_mac,
+    os_is_linux,
+    os_is_pi,
+    has_window_manager,
+    sys_user,
+    get_platform,
+    get_cpu_description,
+    get_gpu_info,
+    get_thermal_info,
+    get_numa_info,
+    get_container_info,
+    get_network_info,
+    get_disk_read_speed,
+    get_cpu_metrics,
+    get_memory_metrics,
+    get_disk_metrics,
+    systeminfo,
+)

cloudmesh/ai/common/aggregation.py

@@ -0,0 +1,130 @@
+"""
+Telemetry aggregation utility for cloudmesh-ai.
+Provides tools to analyze and summarize telemetry data from various backends.
+"""
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Union
+from collections import defaultdict
+
+class TelemetryAggregator:
+    """
+    Analyzes telemetry records to provide summaries and statistics.
+    Supports loading data from both JSONL files and SQLite databases.
+    """
+
+    def __init__(self, source: Union[str, Path]):
+        """
+        Initialize the TelemetryAggregator.
+
+        Args:
+            source: Path to the telemetry source (JSONL file or SQLite .db file).
+        """
+        self.source = Path(source).expanduser()
+        self.records: List[Dict[str, Any]] = []
+        self._load_data()
+
+    def _load_data(self) -> None:
+        """Detects source type and loads records into memory.
+        """
+        if self.source.suffix == ".db":
+            self._load_from_sqlite()
+        else:
+            self._load_from_jsonl()
+
+    def _load_from_jsonl(self) -> None:
+        """Loads records from a JSONL file.
+        """
+        try:
+            if not self.source.exists():
+                return
+            with open(self.source, "r") as f:
+                for line in f:
+                    if line.strip():
+                        self.records.append(json.loads(line))
+        except Exception as e:
+            print(f"Error loading JSONL telemetry: {e}")
+
+    def _load_from_sqlite(self) -> None:
+        """Loads records from a SQLite database.
+        """
+        try:
+            with sqlite3.connect(self.source) as conn:
+                conn.row_factory = sqlite3.Row
+                cursor = conn.execute("SELECT * FROM telemetry")
+                for row in cursor:
+                    record = dict(row)
+                    # Convert JSON strings back to dicts
+                    record["metrics"] = json.loads(record["metrics"]) if isinstance(record["metrics"], str) else record["metrics"]
+                    record["system"] = json.loads(record["system"]) if isinstance(record["system"], str) else record["system"]
+                    self.records.append(record)
+        except Exception as e:
+            print(f"Error loading SQLite telemetry: {e}")
+
+    def get_summary(self) -> Dict[str, Any]:
+        """
+        Calculates a high-level summary of the telemetry data.
+
+        Returns:
+            A dictionary containing total records, success rate, 
+            status distribution, and command distribution.
+        """
+        if not self.records:
+            return {"error": "No records found"}
+
+        total = len(self.records)
+        status_counts = defaultdict(int)
+        command_counts = defaultdict(int)
+        
+        for r in self.records:
+            status_counts[r.get("status", "unknown")] += 1
+            command_counts[r.get("command", "unknown")] += 1
+
+        success_count = status_counts.get("completed", 0)
+        
+        return {
+            "total_records": total,
+            "success_rate": f"{(success_count / total) * 100:.2f}%",
+            "status_distribution": dict(status_counts),
+            "command_distribution": dict(command_counts),
+        }
+
+    def aggregate_metric(self, metric_name: str) -> Dict[str, Any]:
+        """
+        Calculates average, min, and max for a specific metric across all records.
+
+        Args:
+            metric_name: The key of the metric to aggregate from the 'metrics' dictionary.
+
+        Returns:
+            A dictionary containing the count, average, minimum, and maximum values.
+        """
+        values = []
+        for r in self.records:
+            metrics = r.get("metrics", {})
+            if metric_name in metrics:
+                val = metrics[metric_name]
+                if isinstance(val, (int, float)):
+                    values.append(val)
+
+        if not values:
+            return {"metric": metric_name, "status": "no data"}
+
+        return {
+            "metric": metric_name,
+            "count": len(values),
+            "avg": sum(values) / len(values),
+            "min": min(values),
+            "max": max(values),
+        }
+
+if __name__ == "__main__":
+    # Simple test if run directly
+    import sys
+    if len(sys.argv) > 1:
+        agg = TelemetryAggregator(sys.argv[1])
+        print(json.dumps(agg.get_summary(), indent=4))
+    else:
+        print("Usage: python aggregation.py <telemetry_file_or_db>")

cloudmesh/ai/common/io.py

@@ -0,0 +1,101 @@
+"""
+I/O utility functions for cloudmesh-ai.
+Provides helpers for path expansion, YAML handling, and benchmark file creation.
+"""
+
+import os
+import yaml
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+def path_expand(text: str, slashreplace: bool = True) -> str:
+    """Expands a path string by resolving '~', environment variables, and relative links.
+
+    Args:
+        text: The path to be expanded (e.g., "~/$PROJECT/./file.txt").
+        slashreplace: If True, returns backslashes on Windows. Defaults to True.
+
+    Returns:
+        The fully expanded and absolute path.
+    """
+    if not text:
+        return ""
+
+    # 1. Expand ~ and Environment Variables
+    expanded = os.path.expandvars(os.path.expanduser(text))
+    
+    # 2. Convert to a Path object and make it absolute
+    # .resolve() handles the "./" and "../" logic correctly
+    path_obj = Path(expanded).resolve()
+
+    # 3. Handle string conversion and slash preference
+    if slashreplace and os.name == 'nt':
+        # On Windows, this automatically uses backslashes
+        return str(path_obj)
+    
+    # .as_posix() forces forward slashes (/) regardless of OS
+    return path_obj.as_posix()
+
+def load_yaml(path: Path) -> Optional[Dict[str, Any]]:
+    """Safely loads a YAML file from the given path.
+
+    Args:
+        path: Path to the YAML file.
+
+    Returns:
+        The loaded YAML data as a dictionary, or None if the file does not exist 
+        or an error occurs.
+    """
+    try:
+        if not path.exists():
+            return None
+        with open(path, 'r') as f:
+            return yaml.safe_load(f)
+    except (yaml.YAMLError, OSError):
+        return None
+
+def dump_yaml(path: Path, data: Dict[str, Any]) -> None:
+    """Safely writes a dictionary to a YAML file, ensuring the directory exists.
+
+    Args:
+        path: Path where the YAML file should be written.
+        data: The dictionary to write to the file.
+
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'w') as f:
+        yaml.dump(data, f, default_flow_style=False)
+
+def create_benchmark_yaml(path: str, n: int) -> None:
+    """Creates a Cloudmesh service YAML test file with specified number of services.
+
+    Args:
+        path: Path where the benchmark YAML file should be created.
+        n: Number of services to include in the benchmark file.
+
+    """
+    cm = {"cloudmesh": {}}
+    for i in range(0, n):
+        cm["cloudmesh"][f"service{i}"] = {"attribute": f"service{i}"}
+    
+    location = path_expand(path)
+    with open(location, "w") as yaml_file:
+        yaml.dump(cm, yaml_file, default_flow_style=False)
+
+def create_benchmark_file(path: str, n: int) -> int:
+    """Creates a file of a given size in binary megabytes.
+
+    Args:
+        path: Path where the benchmark file should be created.
+        n: Size of the file in megabytes.
+
+    Returns:
+        The actual size of the created file in megabytes.
+    """
+    location = path_expand(path)
+    size = 1048576 * n  # size in bytes
+    with open(location, "wb") as f:
+        f.write(os.urandom(size))
+    
+    s = os.path.getsize(location)
+    return int(s / 1048576.0)

cloudmesh/ai/common/logging.py

@@ -0,0 +1,264 @@
+"""
+Logging utility for cloudmesh-ai components.
+Provides centralized management of log directories, file naming, and logger configuration.
+Includes support for JSON logging, log rotation, and request tracing.
+"""
+
+import logging
+import logging.handlers
+import os
+import sys
+import json
+import threading
+from pathlib import Path
+from datetime import datetime
+from typing import Dict, Optional, Any, Union
+
+# Thread-local storage for request tracing
+_thread_local = threading.local()
+
+def set_context_id(context_id: str):
+    """Sets the context ID for the current thread to enable request tracing.
+
+    Args:
+        context_id: The unique identifier for the current request or context.
+    """
+    _thread_local.context_id = context_id
+
+def get_context_id() -> Optional[str]:
+    """Retrieves the context ID for the current thread.
+
+    Returns:
+        The current context ID if set, otherwise None.
+    """
+    return getattr(_thread_local, "context_id", None)
+
+class ContextFilter(logging.Filter):
+    """Filter that injects the current context_id into the log record."""
+    def filter(self, record: logging.LogRecord) -> bool:
+        """Injects the current context_id into the log record.
+
+        Args:
+            record: The log record to be filtered.
+
+        Returns:
+            True to indicate the record should be logged.
+        """
+        record.context_id = get_context_id() or "system"
+        return True
+
+class JsonFormatter(logging.Formatter):
+    """Formatter that outputs log records in JSON format."""
+    def format(self, record: logging.LogRecord) -> str:
+        """Formats the log record as a JSON string.
+
+        Args:
+            record: The log record to format.
+
+        Returns:
+            A JSON string representation of the log record.
+        """
+        log_record = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "context_id": getattr(record, "context_id", "system"),
+        }
+        if record.exc_info:
+            log_record["exception"] = self.formatException(record.exc_info)
+        return json.dumps(log_record)
+
+# Cache for loggers to prevent duplicate handlers
+_loggers: Dict[str, logging.Logger] = {}
+
+# Global logging configuration
+_logging_config: Dict[str, Any] = {}
+
+def load_logging_config(config_path: Union[str, Path]) -> None:
+    """Loads logging configuration from a JSON file.
+
+    Example config: {"log_dir": "/var/log/cloudmesh", "level": "DEBUG", "json_format": true}
+
+    Args:
+        config_path: Path to the JSON configuration file.
+
+    """
+    global _logging_config
+    try:
+        path = Path(config_path).expanduser()
+        if path.exists():
+            with open(path, "r") as f:
+                _logging_config = json.load(f)
+    except Exception as e:
+        # Use print here because logger might not be configured yet
+        print(f"Failed to load logging config from {config_path}: {e}")
+
+# Automatically load default config on import
+load_logging_config("~/.config/cloudmesh/ai/config.json")
+
+def get_log_dir() -> Path:
+    """Returns the expanded path to the AI logs directory, using config if available.
+
+    Returns:
+        The Path object pointing to the logs directory.
+    """
+    log_dir = _logging_config.get("log_dir", "~/.config/cloudmesh/ai/logs")
+    return Path(log_dir).expanduser()
+
+def ensure_log_dir() -> Path:
+    """Ensures the log directory exists and returns it.
+
+    Returns:
+        The Path object pointing to the ensured logs directory.
+    """
+    log_dir = get_log_dir()
+    log_dir.mkdir(parents=True, exist_ok=True)
+    return log_dir
+
+def get_log_file_path(script_name: str) -> Path:
+    """Generates a timestamped log file path for a given script.
+
+    Example: ~/.config/cloudmesh/ai/logs/test_20260412_124500.log
+
+    Args:
+        script_name: The name of the script for which to generate the log path.
+
+    Returns:
+        The Path object pointing to the generated log file.
+    """
+    log_dir = ensure_log_dir()
+    
+    # Use custom filename if provided in config, otherwise use script_name
+    prefix = _logging_config.get("log_prefix", script_name)
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    return log_dir / f"{prefix}_{timestamp}.log"
+
+def get_logger(
+    script_name: str, 
+    level: Optional[int] = None, 
+    json_format: Optional[bool] = None,
+    max_bytes: Optional[int] = None,
+    backup_count: Optional[int] = None
+) -> logging.Logger:
+    """Returns a configured logger instance for the given script.
+
+    Configures both a rotating file handler and a stream handler.
+    
+    Args:
+        script_name: Name of the logger/script.
+        level: Logging level.
+        json_format: If True, uses JSON formatting for logs.
+        max_bytes: Max size of a log file before rotation.
+        backup_count: Number of backup log files to keep.
+
+    Returns:
+        A configured logging.Logger instance.
+    """
+    if script_name in _loggers:
+        return _loggers[script_name]
+
+    # Resolve settings: Argument > Config File > Default
+    final_level = level if level is not None else _logging_config.get("level", logging.INFO)
+    if isinstance(final_level, str):
+        final_level = getattr(logging, final_level.upper(), logging.INFO)
+
+    final_json = json_format if json_format is not None else _logging_config.get("json_format", False)
+    final_max_bytes = max_bytes if max_bytes is not None else _logging_config.get("max_bytes", 10 * 1024 * 1024)
+    final_backup_count = backup_count if backup_count is not None else _logging_config.get("backup_count", 5)
+
+    logger = logging.getLogger(script_name)
+    logger.setLevel(final_level)
+
+    # Prevent adding handlers if they already exist
+    if not logger.handlers:
+        # Add Context Filter for request tracing
+        logger.addFilter(ContextFilter())
+
+        if final_json:
+            formatter = JsonFormatter()
+        else:
+            formatter = logging.Formatter('%(asctime)s - %(name)s - [%(context_id)s] - %(levelname)s - %(message)s')
+
+        # 1. Rotating File Handler
+        log_file = get_log_file_path(script_name)
+        try:
+            fh = logging.handlers.RotatingFileHandler(
+                log_file, maxBytes=final_max_bytes, backupCount=final_backup_count
+            )
+            fh.setFormatter(formatter)
+            logger.addHandler(fh)
+        except Exception as e:
+            print(f"Failed to initialize rotating file logger at {log_file}: {e}")
+
+        # 2. Stream Handler (Console)
+        sh = logging.StreamHandler()
+        sh.setFormatter(formatter)
+        logger.addHandler(sh)
+
+    _loggers[script_name] = logger
+    return logger
+
+def progress(
+    filename: Optional[str] = None,
+    status: str = "ready",
+    progress: Union[int, str, float] = 0,
+    pid: Optional[Union[int, str]] = None,
+    timestamp: bool = False,
+    stdout: bool = True,
+    stderr: bool = True,
+    append: Optional[str] = None,
+    **kwargs,
+) -> str:
+    """Creates a printed line of the form
+    "# cloudmesh status=ready progress=0 pid=$$ time='2022-08-05 16:29:40.228901'".
+
+    Args:
+        filename: Optional file to append the progress line to.
+        status: The current status string.
+        progress: The current progress value (int, float, or string).
+        pid: Process ID. If None, it will be automatically detected.
+        timestamp: If True, includes a timestamp in the output.
+        stdout: If True, prints to stdout.
+        stderr: If True, prints to stderr.
+        append: Optional string to append to the end of the line.
+        **kwargs: Additional key-value pairs to include in the progress line.
+
+    Returns:
+        The formatted progress string.
+    """
+    if isinstance(progress, (int, float)):
+        progress = str(progress)
+    
+    if pid is None:
+        if "SLURM_JOB_ID" in os.environ:
+            pid = os.environ["SLURM_JOB_ID"]
+        elif "LSB_JOBID" in os.environ:
+            pid = os.environ["LSB_JOBID"]
+        else:
+            pid = os.getpid()
+            
+    variables = ""
+    msg = f"# cloudmesh status={status} progress={progress} pid={pid}"
+    
+    if timestamp:
+        t = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
+        msg = msg + f" time='{t}'"
+        
+    if kwargs:
+        for name, value in kwargs.items():
+            variables = variables + f" {name}={value}"
+        msg = msg + variables
+        
+    if append is not None:
+        msg = msg + " " + append
+        
+    if stdout:
+        print(msg, file=sys.stdout)
+    if stderr:
+        print(msg, file=sys.stderr)
+    if filename is not None:
+        with open(filename, "a") as f:
+            f.write(msg + "\n")
+            
+    return msg