wnm 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

wnm/reports.py

@@ -0,0 +1,286 @@
+"""
+Reports module for weave-node-manager (wnm).
+
+Provides formatted reporting capabilities for node status and details.
+"""
+
+import json
+import logging
+from typing import List, Optional
+
+from sqlalchemy import select
+
+from wnm.models import Node
+from wnm.common import RUNNING, STOPPED, UPGRADING, RESTARTING, REMOVING, DISABLED, DEAD
+from wnm.utils import parse_service_names
+
+
+class NodeReporter:
+    """
+    Reporter class for generating node status reports.
+
+    Supports two report types:
+    - node-status: Tabular summary of nodes
+    - node-status-details: Detailed node information
+    """
+
+    def __init__(self, session_factory):
+        """
+        Initialize reporter with database session factory.
+
+        Args:
+            session_factory: SQLAlchemy scoped_session factory
+        """
+        self.S = session_factory
+        self.logger = logging.getLogger(__name__)
+
+    def _get_nodes(self, service_names: Optional[List[str]] = None) -> List[Node]:
+        """
+        Retrieve nodes from database.
+
+        Args:
+            service_names: Optional list of specific service names to retrieve
+
+        Returns:
+            List of Node objects, ordered appropriately
+        """
+        with self.S() as session:
+            if service_names:
+                # Get specific nodes in the order requested
+                nodes = []
+                for service_name in service_names:
+                    result = session.execute(
+                        select(Node).where(Node.service == service_name)
+                    ).first()
+                    if result:
+                        nodes.append(result[0])
+                    else:
+                        self.logger.warning(f"Node {service_name} not found in database")
+                return nodes
+            else:
+                # Get all nodes ordered by ID (numerical order)
+                results = session.execute(
+                    select(Node).order_by(Node.id)
+                ).all()
+                return [row[0] for row in results]
+
+    def node_status_report(
+        self,
+        service_name: Optional[str] = None,
+        report_format: str = "text"
+    ) -> str:
+        """
+        Generate tabular node status report.
+
+        Format (text):
+            Service Name    Peer ID       Status      Connected Peers
+            antnode0001     12D3Koo...    RUNNING     4
+
+        Format (json):
+            [{"service_name": "antnode0001", "peer_id": "12D3Koo...",
+              "status": "RUNNING", "connected_peers": 4}]
+
+        Args:
+            service_name: Optional comma-separated list of service names
+            report_format: Output format ("text" or "json")
+
+        Returns:
+            Formatted string report
+        """
+        service_names = parse_service_names(service_name)
+        nodes = self._get_nodes(service_names)
+
+        if not nodes:
+            if report_format == "json":
+                return json.dumps({"error": "No nodes found"}, indent=2)
+            return "No nodes found."
+
+        if report_format == "json":
+            # Build JSON output with only the specified fields
+            node_dicts = []
+            for node in nodes:
+                node_dict = {
+                    "service_name": node.service,
+                    "peer_id": node.peer_id or "-",
+                    "status": node.status,
+                    "connected_peers": node.connected_peers if node.connected_peers is not None else 0,
+                }
+                node_dicts.append(node_dict)
+
+            # If single node, return object; if multiple, return array
+            if len(node_dicts) == 1:
+                return json.dumps(node_dicts[0], indent=2)
+            else:
+                return json.dumps(node_dicts, indent=2)
+
+        # Build text report
+        lines = []
+
+        # Header
+        header = f"{'Service Name':<20}{'Peer ID':<55}{'Status':<15}{'Connected Peers':>15}"
+        lines.append(header)
+
+        # Node rows
+        for node in nodes:
+            service_col = f"{node.service:<20}"
+            peer_id_col = f"{(node.peer_id or '-'):<55}"
+            status_col = f"{node.status:<15}"
+            # Connected peers from connected_peers field
+            peers = node.connected_peers if node.connected_peers is not None else 0
+            peers_col = f"{peers:>15}"
+
+            lines.append(f"{service_col}{peer_id_col}{status_col}{peers_col}")
+
+        return "\n".join(lines)
+
+    def node_status_details_report(
+        self,
+        service_name: Optional[str] = None,
+        report_format: str = "text"
+    ) -> str:
+        """
+        Generate detailed node status report.
+
+        Supports two formats:
+        - text: key: value format
+        - json: JSON format with snake_case keys
+
+        Args:
+            service_name: Optional comma-separated list of service names
+            report_format: Output format ("text" or "json")
+
+        Returns:
+            Formatted string report
+        """
+        service_names = parse_service_names(service_name)
+        nodes = self._get_nodes(service_names)
+
+        if not nodes:
+            if report_format == "json":
+                return json.dumps({"error": "No nodes found"}, indent=2)
+            return "No nodes found."
+
+        if report_format == "json":
+            return self._format_details_json(nodes)
+        else:
+            return self._format_details_text(nodes)
+
+    def _format_details_text(self, nodes: List[Node]) -> str:
+        """
+        Format node details as text (key: value format).
+
+        Args:
+            nodes: List of Node objects
+
+        Returns:
+            Formatted text string
+        """
+        sections = []
+
+        for node in nodes:
+            lines = []
+
+            # Service Name
+            lines.append(f"Service Name: {node.service}")
+
+            # Version
+            lines.append(f"Version: {node.version or 'unknown'}")
+
+            # Port
+            lines.append(f"Port: {node.port}")
+
+            # Metrics Port
+            lines.append(f"Metrics Port: {node.metrics_port}")
+
+            # Data path (root_dir)
+            lines.append(f"Data path: {node.root_dir}")
+
+            # Log path - construct from root_dir
+            log_path = f"{node.root_dir}/logs" if node.root_dir else "unknown"
+            lines.append(f"Log path: {log_path}")
+
+            # Bin path - construct from root_dir and binary name
+            bin_path = f"{node.root_dir}/{node.binary}" if node.root_dir and node.binary else "unknown"
+            lines.append(f"Bin Path: {bin_path}")
+
+            # Connected peers from connected_peers field
+            connected_peers = node.connected_peers if node.connected_peers is not None else 0
+            lines.append(f"Connected peers: {connected_peers}")
+
+            # Rewards address from node's wallet field
+            rewards_address = node.wallet or "unknown"
+            lines.append(f"Rewards address: {rewards_address}")
+
+            # Age in seconds
+            age_seconds = node.age if node.age is not None else 0
+            lines.append(f"Age: {age_seconds}")
+
+            # Peer ID
+            lines.append(f"Peer ID: {node.peer_id or '-'}")
+
+            # Status
+            lines.append(f"Status: {node.status}")
+
+            sections.append("\n".join(lines))
+
+        # Separate multiple nodes with blank line
+        return "\n\n".join(sections)
+
+    def _format_details_json(self, nodes: List[Node]) -> str:
+        """
+        Format node details as JSON using snake_case field names from model.
+
+        Args:
+            nodes: List of Node objects
+
+        Returns:
+            JSON formatted string
+        """
+        # Use the __json__ method from the Node model
+        node_dicts = [node.__json__() for node in nodes]
+
+        # If single node, return object; if multiple, return array
+        if len(node_dicts) == 1:
+            return json.dumps(node_dicts[0], indent=2)
+        else:
+            return json.dumps(node_dicts, indent=2)
+
+
+def generate_node_status_report(
+    session_factory,
+    service_name: Optional[str] = None,
+    report_format: str = "text"
+) -> str:
+    """
+    Convenience function to generate node status report.
+
+    Args:
+        session_factory: SQLAlchemy scoped_session factory
+        service_name: Optional comma-separated list of service names
+        report_format: Output format ("text" or "json")
+
+    Returns:
+        Formatted report string
+    """
+    reporter = NodeReporter(session_factory)
+    return reporter.node_status_report(service_name, report_format)
+
+
+def generate_node_status_details_report(
+    session_factory,
+    service_name: Optional[str] = None,
+    report_format: str = "text"
+) -> str:
+    """
+    Convenience function to generate node status details report.
+
+    Args:
+        session_factory: SQLAlchemy scoped_session factory
+        service_name: Optional comma-separated list of service names
+        report_format: Output format ("text" or "json")
+
+    Returns:
+        Formatted report string
+    """
+    reporter = NodeReporter(session_factory)
+    return reporter.node_status_details_report(service_name, report_format)

wnm/utils.py

@@ -0,0 +1,403 @@
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+from collections import Counter
+from typing import List, Optional
+
+import psutil
+import requests
+from sqlalchemy import create_engine, delete, insert, select, text, update
+from sqlalchemy.orm import scoped_session, sessionmaker
+
+from wnm.common import (
+    DEAD,
+    DISABLED,
+    DONATE,
+    METRICS_PORT_BASE,
+    MIGRATING,
+    MIN_NODES_THRESHOLD,
+    PORT_MULTIPLIER,
+    QUEEN,
+    REMOVING,
+    RESTARTING,
+    RUNNING,
+    STOPPED,
+    UPGRADING,
+)
+from wnm.config import BOOTSTRAP_CACHE_DIR, LOG_DIR, PLATFORM
+from wnm.models import Base, Machine, Node
+
+
+def parse_service_names(service_name_str: Optional[str]) -> Optional[List[str]]:
+    """Parse comma-separated service names.
+
+    Args:
+        service_name_str: Comma-separated service names (e.g., "antnode0001,antnode0003")
+
+    Returns:
+        List of service names, or None if input is None/empty
+    """
+    if not service_name_str:
+        return None
+
+    # Split by comma and strip whitespace
+    names = [name.strip() for name in service_name_str.split(',')]
+    # Filter out empty strings
+    return [name for name in names if name]
+
+
+# Read config from systemd service file
+def read_node_metadata(host, port):
+    # Only return version number when we have one, to stop clobbering the binary check
+    try:
+        url = "http://{0}:{1}/metadata".format(host, port)
+        response = requests.get(url, timeout=5)
+        data = response.text
+    except requests.exceptions.ConnectionError:
+        logging.debug("Connection Refused on port: {0}:{1}".format(host, str(port)))
+        return {"status": STOPPED, "peer_id": ""}
+    except Exception as error:
+        template = "In RNMd - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.info(message)
+        return {"status": STOPPED, "peer_id": ""}
+    # collect a dict to return
+    card = {}
+    try:
+        card["version"] = re.findall(r'{antnode_version="([\d\.]+)"}', data)[0]
+    except (IndexError, KeyError) as e:
+        logging.info(f"No version found: {e}")
+    try:
+        card["peer_id"] = re.findall(r'{peer_id="([\w\d]+)"}', data)[0]
+    except (IndexError, KeyError) as e:
+        logging.debug(f"No peer_id found: {e}")
+        card["peer_id"] = ""
+    card["status"] = RUNNING if "version" in card else STOPPED
+    return card
+
+
+# Read data from metrics port
+def read_node_metrics(host, port):
+    metrics = {}
+    try:
+        url = "http://{0}:{1}/metrics".format(host, port)
+        response = requests.get(url, timeout=5)
+        metrics["status"] = RUNNING
+        metrics["uptime"] = int(
+            (re.findall(r"ant_node_uptime ([\d]+)", response.text) or [0])[0]
+        )
+        metrics["records"] = int(
+            (
+                re.findall(r"ant_networking_records_stored ([\d]+)", response.text)
+                or [0]
+            )[0]
+        )
+        metrics["shunned"] = int(
+            (
+                re.findall(
+                    r"ant_networking_shunned_by_close_group ([\d]+)", response.text
+                )
+                or [0]
+            )[0]
+        )
+        metrics["connected_peers"] = int(
+            (
+                re.findall(r"ant_networking_connected_peers ([\d]+)", response.text)
+                or [0]
+            )[0]
+        )
+    except requests.exceptions.ConnectionError:
+        logging.debug("Connection Refused on port: {0}:{1}".format(host, str(port)))
+        metrics["status"] = STOPPED
+        metrics["uptime"] = 0
+        metrics["records"] = 0
+        metrics["shunned"] = 0
+        metrics["connected_peers"] = 0
+    except Exception as error:
+        template = "in:RNM - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.info(message)
+        metrics["status"] = STOPPED
+        metrics["uptime"] = 0
+        metrics["records"] = 0
+        metrics["shunned"] = 0
+        metrics["connected_peers"] = 0
+    return metrics
+
+
+# Read antnode binary version
+def get_antnode_version(binary):
+    try:
+        data = subprocess.run(
+            [binary, "--version"], stdout=subprocess.PIPE
+        ).stdout.decode("utf-8")
+        return re.findall(r"Autonomi Node v([\d\.]+)", data)[0]
+    except Exception as error:
+        template = "In GAV - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.info(message)
+        return 0
+
+
+# Determine how long this node has been around by looking at it's secret_key file
+def get_node_age(root_dir):
+    try:
+        return int(os.stat("{0}/secret-key".format(root_dir)).st_mtime)
+    except (FileNotFoundError, OSError) as e:
+        logging.debug(f"Unable to get node age for {root_dir}: {e}")
+        return 0
+
+
+# Survey nodes by reading metadata from metrics ports or binary --version
+def get_machine_metrics(S, node_storage, remove_limit, crisis_bytes):
+    metrics = {}
+
+    with S() as session:
+        db_nodes = session.execute(select(Node.status, Node.version)).all()
+
+    # Get system start time before we probe metrics
+    try:
+        if PLATFORM == "Darwin":
+            # macOS: use sysctl kern.boottime
+            p = subprocess.run(
+                ["sysctl", "-n", "kern.boottime"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=True,
+            ).stdout.decode("utf-8")
+            # Parse: { sec = 1234567890, usec = 0 }
+            match = re.search(r"sec = (\d+)", p)
+            if match:
+                metrics["system_start"] = int(match.group(1))
+            else:
+                raise ValueError("Could not parse kern.boottime")
+        else:
+            # Linux: use uptime --since
+            p = subprocess.run(
+                ["uptime", "--since"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            ).stdout.decode("utf-8")
+            if re.match(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}", p):
+                metrics["system_start"] = int(
+                    time.mktime(time.strptime(p.strip(), "%Y-%m-%d %H:%M:%S"))
+                )
+    except (subprocess.CalledProcessError, ValueError) as err:
+        logging.error("GMM ERROR:", err)
+        metrics["system_start"] = 0
+
+    # Get some initial stats for comparing after a few seconds
+    # We start these counters AFTER reading the database
+    start_time = time.time()
+    start_disk_counters = psutil.disk_io_counters()
+    start_net_counters = psutil.net_io_counters()
+
+    metrics["total_nodes"] = len(db_nodes)
+    data = Counter(node[0] for node in db_nodes)
+    metrics["running_nodes"] = data[RUNNING]
+    metrics["stopped_nodes"] = data[STOPPED]
+    metrics["restarting_nodes"] = data[RESTARTING]
+    metrics["upgrading_nodes"] = data[UPGRADING]
+    metrics["migrating_nodes"] = data[MIGRATING]
+    metrics["removing_nodes"] = data[REMOVING]
+    metrics["dead_nodes"] = data[DEAD]
+    metrics["antnode"] = shutil.which("antnode")
+    if not metrics["antnode"]:
+        logging.warning("Unable to locate current antnode binary, exiting")
+        sys.exit(1)
+    metrics["antnode_version"] = get_antnode_version(metrics["antnode"])
+    metrics["queen_node_version"] = (
+        db_nodes[0][1] if metrics["total_nodes"] > 0 else metrics["antnode_version"]
+    )
+    metrics["nodes_latest_v"] = (
+        sum(1 for node in db_nodes if node[1] == metrics["antnode_version"]) or 0
+    )
+    metrics["nodes_no_version"] = sum(1 for node in db_nodes if not node[1]) or 0
+    metrics["nodes_to_upgrade"] = (
+        metrics["total_nodes"] - metrics["nodes_latest_v"] - metrics["nodes_no_version"]
+    )
+    metrics["nodes_by_version"] = Counter(ver[1] for ver in db_nodes)
+
+    # Windows has to build load average over 5 seconds. The first 5 seconds returns 0's
+    # I don't plan on supporting windows, but if this get's modular, I don't want this
+    # issue to be skipped
+    # if platform.system() == "Windows":
+    #    discard=psutil.getloadavg()
+    #    time.sleep(5)
+    metrics["load_average_1"], metrics["load_average_5"], metrics["load_average_15"] = (
+        psutil.getloadavg()
+    )
+    # Get CPU Metrics over 1 second
+    cpu_times = psutil.cpu_times_percent(1)
+    if PLATFORM == "Darwin":
+        # macOS: cpu_times has (user, nice, system, idle) - no iowait
+        metrics["idle_cpu_percent"] = cpu_times.idle
+        metrics["io_wait"] = 0  # Not available on macOS
+    else:
+        # Linux: cpu_times has (user, nice, system, idle, iowait, ...)
+        metrics["idle_cpu_percent"], metrics["io_wait"] = cpu_times[3:5]
+    # Really we returned Idle percent, subtract from 100 to get used.
+    metrics["used_cpu_percent"] = 100 - metrics["idle_cpu_percent"]
+    data = psutil.virtual_memory()
+    # print(data)
+    metrics["used_mem_percent"] = data.percent
+    metrics["free_mem_percent"] = 100 - metrics["used_mem_percent"]
+    data = psutil.disk_io_counters()
+    # This only checks the drive mapped to the first node and will need to be updated
+    # when we eventually support multiple drives
+    data = psutil.disk_usage(node_storage)
+    metrics["used_hd_percent"] = data.percent
+    metrics["total_hd_bytes"] = data.total
+    end_time = time.time()
+    end_disk_counters = psutil.disk_io_counters()
+    end_net_counters = psutil.net_io_counters()
+    metrics["hdio_write_bytes"] = int(
+        (end_disk_counters.write_bytes - start_disk_counters.write_bytes)
+        / (end_time - start_time)
+    )
+    metrics["hdio_read_bytes"] = int(
+        (end_disk_counters.read_bytes - start_disk_counters.read_bytes)
+        / (end_time - start_time)
+    )
+    metrics["netio_write_bytes"] = int(
+        (end_net_counters.bytes_sent - start_net_counters.bytes_sent)
+        / (end_time - start_time)
+    )
+    metrics["netio_read_bytes"] = int(
+        (end_net_counters.bytes_recv - start_net_counters.bytes_recv)
+        / (end_time - start_time)
+    )
+    # print (json.dumps(metrics,indent=2))
+    # How close (out of 100) to removal limit will we be with a max bytes per node (2GB default)
+    # For running nodes with Porpoise(tm).
+    metrics["node_hd_crisis"] = int(
+        (
+            ((metrics["total_nodes"]) * int(crisis_bytes))
+            / (metrics["total_hd_bytes"] * (remove_limit / 100))
+        )
+        * 100
+    )
+    return metrics
+
+
+# Update node with metrics result
+def update_node_from_metrics(S, id, metrics, metadata):
+    try:
+        # We check the binary version in other code, so lets stop clobbering it when a node is stopped
+        card = {
+            "status": metrics["status"],
+            "timestamp": int(time.time()),
+            "uptime": metrics["uptime"],
+            "records": metrics["records"],
+            "shunned": metrics["shunned"],
+            "connected_peers": metrics["connected_peers"],
+            "peer_id": metadata["peer_id"],
+        }
+        if "version" in metadata:
+            card["version"] = metadata["version"]
+        with S() as session:
+            session.query(Node).filter(Node.id == id).update(card)
+            session.commit()
+    except Exception as error:
+        template = "In UNFM - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.warning(message)
+        return False
+    else:
+        return True
+
+
+# Set Node status
+def update_counters(S, old, config):
+    # Are we already removing a node
+    if old["removing_nodes"]:
+        with S() as session:
+            removals = session.execute(
+                select(Node.timestamp, Node.id)
+                .where(Node.status == REMOVING)
+                .order_by(Node.timestamp.asc())
+            ).all()
+        # Iterate through active removals
+        records_to_remove = len(removals)
+        for check in removals:
+            # If the delay_remove timer has expired, delete the entry
+            if isinstance(check[0], int) and check[0] < (
+                int(time.time()) - config["delay_remove"]
+            ):
+                logging.info("Deleting removed node " + str(check[1]))
+                with S() as session:
+                    session.execute(delete(Node).where(Node.id == check[1]))
+                    session.commit()
+                records_to_remove -= 1
+        old["removing_nodes"] = records_to_remove
+    # Are we already upgrading a node
+    if old["upgrading_nodes"]:
+        with S() as session:
+            upgrades = session.execute(
+                select(Node.timestamp, Node.id, Node.host, Node.metrics_port)
+                .where(Node.status == UPGRADING)
+                .order_by(Node.timestamp.asc())
+            ).all()
+        # Iterate through active upgrades
+        records_to_upgrade = len(upgrades)
+        for check in upgrades:
+            # If the delay_upgrade timer has expired, check on status
+            if isinstance(check[0], int) and check[0] < (
+                int(time.time()) - config["delay_upgrade"]
+            ):
+                logging.info("Updating upgraded node " + str(check[1]))
+                node_metrics = read_node_metrics(check[2], check[3])
+                node_metadata = read_node_metadata(check[2], check[3])
+                if node_metrics and node_metadata:
+                    update_node_from_metrics(S, check[1], node_metrics, node_metadata)
+                records_to_upgrade -= 1
+        old["upgrading_nodes"] = records_to_upgrade
+    # Are we already restarting a node
+    if old["restarting_nodes"]:
+        with S() as session:
+            restarts = session.execute(
+                select(Node.timestamp, Node.id, Node.host, Node.metrics_port)
+                .where(Node.status == RESTARTING)
+                .order_by(Node.timestamp.asc())
+            ).all()
+        # Iterate through active upgrades
+        records_to_restart = len(restarts)
+        for check in restarts:
+            # If the delay_start timer has expired, check on status
+            if isinstance(check[0], int) and check[0] < (
+                int(time.time()) - config["delay_start"]
+            ):
+                logging.info("Updating restarted node " + str(check[1]))
+                node_metrics = read_node_metrics(check[2], check[3])
+                node_metadata = read_node_metadata(check[2], check[3])
+                if node_metrics and node_metadata:
+                    update_node_from_metrics(S, check[1], node_metrics, node_metadata)
+                records_to_restart -= 1
+        old["restarting_nodes"] = records_to_restart
+    return old
+
+
+# Enable firewall for port
+def update_nodes(S):
+    with S() as session:
+        nodes = session.execute(
+            select(Node.timestamp, Node.id, Node.host, Node.metrics_port, Node.status)
+            .where(Node.status != DISABLED)
+            .order_by(Node.timestamp.asc())
+        ).all()
+    # Iterate through all records
+    for check in nodes:
+        # Check on status
+        if isinstance(check[0], int):
+            logging.debug("Updating info on node " + str(check[1]))
+            node_metrics = read_node_metrics(check[2], check[3])
+            node_metadata = read_node_metadata(check[2], check[3])
+            if node_metrics and node_metadata:
+                # Don't write updates for stopped nodes that are already marked as stopped
+                if node_metadata["status"] == STOPPED and check[4] == STOPPED:
+                    continue
+                update_node_from_metrics(S, check[1], node_metrics, node_metadata)