wnm 0.0.4__py3-none-any.whl

wnm/__init__.py

@@ -0,0 +1,3 @@
+"""A service to manage a cluster of decentralized Autonomi nodes"""
+
+__version__ = "0.0.4"

wnm/__main__.py

@@ -0,0 +1,976 @@
+import os, sys
+import re, json, requests, time
+import subprocess, logging
+from collections import Counter
+from packaging.version import Version
+from dotenv import load_dotenv
+import psutil, shutil
+
+from wnm.models import Base, Machine, Node
+from sqlalchemy import create_engine, select, insert, update, delete, text
+from sqlalchemy.orm import sessionmaker, scoped_session
+
+logging.basicConfig(level=logging.INFO)
+#Info level logging for sqlalchemy is too verbose, only use when needed
+logging.getLogger('sqlalchemy.engine.Engine').disabled = True
+        
+# import .env
+basedir = os.path.abspath(os.path.dirname(__file__))
+load_dotenv(os.path.join(basedir, '.env'))
+
+# simulate arg/yaml configuration
+config = {}
+config['db']='sqlite:///colony.db'
+config['DonateAddress'] = os.getenv('DonateAddress') or '0x00455d78f850b0358E8cea5be24d415E01E107CF'
+config['ANMHost'] = os.getenv('ANMHost') or '127.0.0.1'
+config['CrisisBytes'] = os.getenv('CrisisBytes') or 2 * 10 ** 9 # default 2gb/node
+
+
+# Setup Database engine
+engine = create_engine(config["db"], echo=True)
+
+# Generate ORM
+Base.metadata.create_all(engine)
+
+# Create a connection to the ORM
+session_factory = sessionmaker(bind=engine)
+S = scoped_session(session_factory)
+
+
+# if WNM_CONFIG or -c parameter are set, check for existing config
+# else:
+
+# Primary node for want of one
+QUEEN=1
+
+# Donation address
+DONATE=config["DonateAddress"]
+#Keep these as strings so they can be grepped in logs
+STOPPED="STOPPED" #0 Node is not responding to it's metrics port
+RUNNING="RUNNING" #1 Node is responding to it's metrics port
+UPGRADING="UPGRADING" #2 Upgrade in progress
+DISABLED="DISABLED" #-1 Do not start
+RESTARTING="RESTARTING" #3 re/starting a server intionally
+MIGRATING="MIGRATING" #4 Moving volumes in progress
+REMOVING="REMOVING" #5 Removing node in progress
+DEAD="DEAD" #-86 Broken node to cleanup
+
+ANM_HOST=config["ANMHost"]
+# Baseline bytes per node
+CRISIS_BYTES=config["CrisisBytes"]
+
+# A storage place for ant node data
+Workers=[]
+
+# Detect ANM (but don't upgrade)
+if os.path.exists("/var/antctl/system"):
+    # Is anm scheduled to run
+    if os.path.exists("/etc/cron.d/anm"):
+        # remove cron to disable old anm
+        os.remove("/etc/cron.d/anm")
+    # Is anm sitll running? We'll wait
+    if os.path.exists("/var/antctl/block"):
+        logging.info("anm still running, waiting...")
+        sys.exit(1)
+
+# Are we already running
+if os.path.exists("/var/antctl/wnm_active"):
+    logging.info("wnm still running")
+    sys.exit(1)
+
+# Get anm configuration
+def load_anm_config():
+    anm_config = {}
+
+    # Let's get the real count of CPU's available to this process
+    anm_config["CpuCount"] = len(os.sched_getaffinity(0))
+
+    # What can we save from /var/antctl/config
+    if os.path.exists("/var/antctl/config"):
+        load_dotenv("/var/antctl/config")
+    anm_config["NodeCap"] = int(os.getenv('NodeCap') or 20)
+    anm_config["CpuLessThan"] = int(os.getenv('CpuLessThan') or 50)
+    anm_config["CpuRemove"] = int(os.getenv('CpuRemove') or 70)
+    anm_config["MemLessThan"] = int(os.getenv('MemLessThan') or 70)
+    anm_config["MemRemove"] = int(os.getenv('MemRemove') or 90)
+    anm_config["HDLessThan"] = int(os.getenv('HDLessThan') or 70)
+    anm_config["HDRemove"] = int(os.getenv('HDRemove') or 90)
+    anm_config["DelayStart"] = int(os.getenv('DelayStart') or 5)
+    anm_config["DelayUpgrade"] = int(os.getenv('DelayUpgrade') or 5)
+    anm_config["DelayRestart"] = int(os.getenv('DelayRestart') or 10)
+    anm_config["DelayRemove"] = int(os.getenv('DelayRemove') or 300)
+    anm_config["NodeStorage"] = os.getenv('NodeStorage') or "/var/antctl/services"
+    # Default to the faucet donation address
+    try:
+        anm_config["RewardsAddress"] = re.findall(r"--rewards-address ([\dA-Fa-fXx]+)",os.getenv('RewardsAddress'))[0]
+    except:
+        try:
+            anm_config["RewardsAddress"] = re.findall(r"([\dA-Fa-fXx]+)",os.getenv("RewardsAddress"))[0]
+        except:
+            logging.warning("Unable to detect RewardsAddress")
+            sys.exit(1)
+    anm_config["DonateAddress"]=os.getenv("DonateAddress") or DONATE
+    anm_config["MaxLoadAverageAllowed"]=float(os.getenv("MaxLoadAverageAllowed") or anm_config["CpuCount"])
+    anm_config["DesiredLoadAverage"]=float(os.getenv("DesiredLoadAverage") or (anm_config["CpuCount"] * .6))
+
+    try:
+        with open('/usr/bin/anms.sh', 'r') as file:
+            data = file.read()
+        anm_config["PortStart"]=int(re.findall(r"ntpr\=(\d+)",data)[0])
+    except:
+        anm_config["PortStart"]=55
+
+    anm_config["HDIOReadLessThan"] = float(os.getenv('HDIOReadLessThan') or 0.0)
+    anm_config["HDIOReadRemove"] = float(os.getenv('HDIOReadRemove') or 0.0)
+    anm_config["HDIOWriteLessThan"] = float(os.getenv('HDIOWriteLessThan') or 0.0)
+    anm_config["HDIOWriteRemove"] = float(os.getenv('HDIOWriteRemove') or 0.0)
+    anm_config["NetIOReadLessThan"] = float(os.getenv('NetIOReadLessThan') or 0.0)
+    anm_config["NetIOReadRemove"] = float(os.getenv('NetIOReadRemove') or 0.0)
+    anm_config["NetIOWriteLessThan"] = float(os.getenv('NetIOWriteLessThan') or 0.0)
+    anm_config["NetIOWriteRemove"] = float(os.getenv('NetIOWriteRemove') or 0.0)
+    # Timer for last stopped nodes
+    anm_config["LastStoppedAt"]=0
+
+
+    return anm_config
+
+# Read confirm from systemd service file
+def read_systemd_service(antnode):
+    details={}
+    try:
+        with open('/etc/systemd/system/'+antnode, 'r') as file:
+            data = file.read()
+        details['id']=int(re.findall(r"antnode(\d+)",antnode)[0])
+        details['binary']=re.findall(r"ExecStart=([^ ]+)",data)[0]
+        details["user"]=re.findall(r"User=(\w+)",data)[0]
+        details["root_dir"]=re.findall(r"--root-dir ([\w\/]+)",data)[0]
+        details["port"]=int(re.findall(r"--port (\d+)",data)[0])
+        details["metrics_port"]=int(re.findall(r"--metrics-server-port (\d+)",data)[0])
+        details["wallet"]=re.findall(r"--rewards-address ([^ ]+)",data)[0]
+        details["network"]=re.findall(r"--rewards-address [^ ]+ ([\w\-]+)",data)[0]
+    except:
+        pass
+    
+    return details
+
+# Read data from metadata endpoint
+def read_node_metadata(host,port):
+    # Only return version number when we have one, to stop clobbering the binary check
+    try:
+        url = "http://{0}:{1}/metadata".format(host,port)
+        response = requests.get(url)
+        data=response.text
+    except requests.exceptions.ConnectionError:
+        logging.debug("Connection Refused on port: {0}:{1}".format(host,str(port)))
+        return {"status": STOPPED, "peer_id":""}
+    except Exception as error:
+        template = "In RNMd - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.info(message)
+        return {"status": STOPPED, "peer_id":""}
+    # collect a dict to return
+    card={}
+    try:
+        card["version"] = re.findall(r'{antnode_version="([\d\.]+)"}',data)[0]
+    except:
+        logging.info('No version found')
+    try:
+        card["peer_id"] = re.findall(r'{peer_id="([\w\d]+)"}',data)[0]
+    except:
+        card["peer_id"] = ""
+    card["status"] = RUNNING if "version" in card else STOPPED
+    return card
+
+# Read data from metrics port
+def read_node_metrics(host,port):
+    metrics={}
+    try:
+        url = "http://{0}:{1}/metrics".format(host,port)
+        response = requests.get(url)
+        metrics["status"] = RUNNING
+        metrics["uptime"] = int((re.findall(r'ant_node_uptime ([\d]+)',response.text) or [0])[0])
+        metrics["records"] = int((re.findall(r'ant_networking_records_stored ([\d]+)',response.text) or [0])[0])
+        metrics["shunned"] = int((re.findall(r'ant_networking_shunned_by_close_group ([\d]+)',response.text) or [0])[0])
+    except requests.exceptions.ConnectionError:
+        logging.debug("Connection Refused on port: {0}:{1}".format(host,str(port)))
+        metrics["status"] = STOPPED
+        metrics["uptime"] = 0
+        metrics["records"] = 0
+        metrics["shunned"] = 0
+    except Exception as error:
+        template = "in:RNM - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.info(message)
+        metrics["status"] = STOPPED
+        metrics["uptime"] = 0
+        metrics["records"] = 0
+        metrics["shunned"] = 0
+    return metrics
+    
+# Read antnode binary version
+def get_antnode_version(binary):
+    try:
+        data = subprocess.run([binary, '--version'], stdout=subprocess.PIPE).stdout.decode('utf-8')
+        return re.findall(r'Autonomi Node v([\d\.]+)',data)[0]
+    except Exception as error:
+        template = "In GAV - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.info(message)
+        return 0
+    
+# Determine how long this node has been around by looking at it's secret_key file
+def get_node_age(root_dir):
+    try:
+        return int(os.stat("{0}/secret-key".format(root_dir)).st_mtime)
+    except:
+        return 0
+    
+# Survey nodes by reading metadata from metrics ports or binary --version
+def survey_anm_nodes(antnodes):
+    # Build a list of node dictionaries to return
+    details=[]
+    # Iterate on nodes
+    for node in antnodes:
+        # Initialize a dict
+        logging.debug("{0} surveying node {1} ".format(time.strftime("%Y-%m-%d %H:%M"),node))
+        if not re.findall(r"antnode([\d]+).service",node):
+            logging.info("can't decode "+str(node))
+            continue
+        card={"nodename":re.findall(r"antnode([\d]+).service",node)[0],
+              "service": node,
+              "timestamp": int(time.time()),
+              "host": ANM_HOST or '127.0.0.1'
+              }
+        # Load what systemd has configured
+        card.update(read_systemd_service(node))
+        #print(json.dumps(card,indent=2))
+        # Read metadata from metrics_port
+        metadata = read_node_metadata(card["host"],card["metrics_port"])
+        #print(json.dumps(metadata,indent=2))
+        if  isinstance(metadata,dict) and \
+            "status" in metadata and \
+            metadata["status"]==RUNNING:
+            # soak up metadata
+            card.update(metadata)
+            # The ports up, so grab metrics too
+            card.update(read_node_metrics(card["host"],card["metrics_port"]))
+        # Else run binary to get version
+        else:
+            # If the root directory of the node is missing, it's a bad node
+            if not os.path.isdir(card["root_dir"]):
+                card["status"]=DEAD
+                card["version"]=''
+            else:
+                card["status"]=STOPPED
+                card["version"]=get_antnode_version(card["binary"])
+            card["peer_id"]=''
+            card["records"]=0
+            card["uptime"]=0
+            card["shunned"]=0
+        card["age"]=get_node_age(card["root_dir"])
+        # harcoded for anm
+        card["host"]=ANM_HOST
+        # Append the node dict to the detail list
+        details.append(card)
+    
+    return details
+
+# Survey server instance
+def survey_machine():
+    # Make a bucket
+    antnodes=[]
+    # For all service files
+    for file in os.listdir("/etc/systemd/system"):
+        # Find antnodes
+        if re.match(r'antnode[\d]+\.service',file):
+            antnodes.append(file)
+        #if len(antnodes)>=5:
+        #   break
+    # Iterate over defined nodes and get details
+    # Ingests a list of service files and outputs a list of dictionaries
+    return survey_anm_nodes(antnodes)
+
+# Read system status
+def get_machine_metrics(node_storage,remove_limit):
+    metrics = {}
+
+    with S() as session:
+        db_nodes=session.execute(select(Node.status,Node.version)).all()
+    
+    # Get some initial stats for comparing after a few seconds
+    # We start these counters AFTER reading the database
+    start_time=time.time()
+    start_disk_counters=psutil.disk_io_counters()
+    start_net_counters=psutil.net_io_counters()
+
+    metrics["TotalNodes"]=len(db_nodes)
+    data = Counter(node[0] for node in db_nodes)
+    metrics["RunningNodes"] = data[RUNNING]
+    metrics["StoppedNodes"] = data[STOPPED]
+    metrics["RestartingNodes"] = data[RESTARTING]
+    metrics["UpgradingNodes"] = data[UPGRADING]
+    metrics["MigratingNodes"] = data[MIGRATING]
+    metrics["RemovingNodes"] = data[REMOVING]
+    metrics["DeadNodes"] = data[DEAD]
+    metrics["antnode"]=shutil.which("antnode")
+    if not metrics["antnode"]:
+        logging.warning("Unable to locate current antnode binary, exiting")
+        sys.exit(1)
+    metrics["AntNodeVersion"]=get_antnode_version(metrics["antnode"])
+    metrics["NodesLatestV"]=sum(1 for node in db_nodes if node[1]==metrics["AntNodeVersion"]) or 0
+    metrics["NodesNoVersion"]=sum(1 for node in db_nodes if not node[1]) or 0
+    metrics["NodesToUpgrade"]=metrics["TotalNodes"] - metrics["NodesLatestV"] - metrics["NodesNoVersion"]
+
+    # Windows has to build load average over 5 seconds. The first 5 seconds returns 0's
+    # I don't plan on supporting windows, but if this get's modular, I don't want this 
+    # issue to be skipped
+    #if platform.system() == "Windows":
+    #    discard=psutil.getloadavg()
+    #    time.sleep(5)
+    metrics["LoadAverage1"],metrics["LoadAverage5"],metrics["LoadAverage15"]=psutil.getloadavg()
+    # Get CPU Metrics over 1 second
+    metrics["IdleCpuPercent"],metrics["IOWait"] = psutil.cpu_times_percent(1)[3:5]
+    # Really we returned Idle percent, subtract from 100 to get used.
+    metrics["UsedCpuPercent"] = 100 - metrics["IdleCpuPercent"]
+    data=psutil.virtual_memory()
+    #print(data)
+    metrics["UsedMemPercent"]=data.percent
+    metrics["FreeMemPercent"]=100-metrics["UsedMemPercent"]
+    data=psutil.disk_io_counters()
+    # This only checks the drive mapped to the first node and will need to be updated
+    # when we eventually support multiple drives
+    data=psutil.disk_usage(node_storage)
+    metrics["UsedHDPercent"]=data.percent
+    metrics["TotalHDBytes"]=data.total
+    end_time=time.time()
+    end_disk_counters=psutil.disk_io_counters()
+    end_net_counters=psutil.net_io_counters()
+    metrics["HDWriteBytes"]=int((end_disk_counters.write_bytes-start_disk_counters.write_bytes)/(end_time-start_time))
+    metrics["HDReadBytes"]=int((end_disk_counters.read_bytes-start_disk_counters.read_bytes)/(end_time-start_time))
+    metrics["NetWriteBytes"]=int((end_net_counters.bytes_sent-start_net_counters.bytes_sent)/(end_time-start_time))
+    metrics["NetReadBytes"]=int((end_net_counters.bytes_recv-start_net_counters.bytes_recv)/(end_time-start_time))
+    #print (json.dumps(metrics,indent=2))
+    # How close (out of 100) to removal limit will we be with a max bytes per node (2GB default)
+    # For running nodes with Porpoise(tm).
+    metrics["NodeHDCrisis"]=int((((metrics["TotalNodes"])*CRISIS_BYTES)/(metrics["TotalHDBytes"]*(remove_limit/100)))*100)
+    return metrics
+
+# Update node with metrics result
+def update_node_from_metrics(id,metrics,metadata):
+    try:
+        # We check the binary version in other code, so lets stop clobbering it when a node is stopped
+        card={'status': metrics["status"], 'timestamp': int(time.time()),
+                        'uptime': metrics["uptime"], 'records': metrics["records"],
+                        'shunned': metrics["shunned"],
+                        'peer_id': metadata["peer_id"]}
+        if "version" in metadata:
+            card['version']=metadata["version"]
+        with S() as session:
+            session.query(Node).filter(Node.id == id).\
+                update(card)
+            session.commit()
+    except Exception as error:
+        template = "In UNFM - An exception of type {0} occurred. Arguments:\n{1!r}"
+        message = template.format(type(error).__name__, error.args)
+        logging.warning(message)
+        return False
+    else:
+        return True
+    
+# Set Node status
+def set_node_status(id,status):
+    logging.info("Setting node status: {0} {1}".format(id,status))
+    try:
+        with S() as session:
+            session.query(Node).filter(Node.id == id).\
+                update({'status': status, 'timestamp': int(time.time())})
+            session.commit()
+    except:
+        return False
+    else:
+        return True
+
+# Update metrics after checking counters
+def update_counters(old,config):
+    # Are we already removing a node
+    if old["RemovingNodes"]:
+        with S() as session:
+            removals=session.execute(select(Node.timestamp,Node.id)\
+                                         .where(Node.status == REMOVING)\
+                                         .order_by(Node.timestamp.asc())).all()
+        # Iterate through active removals
+        records_to_remove = len(removals)
+        for check in removals:
+            # If the DelayRemove timer has expired, delete the entry
+            if isinstance(check[0],int) and \
+                check[0] < (int(time.time()) - (config["DelayRemove"]*60)):
+                logging.info("Deleting removed node "+str(check[1]))
+                with S() as session:
+                    session.execute(delete(Node).where(Node.id==check[1]))
+                    session.commit()
+                records_to_remove-=1
+        old["RemovingNodes"]=records_to_remove
+    # Are we already upgrading a node
+    if old["UpgradingNodes"]:
+        with S() as session:
+            upgrades=session.execute(select(Node.timestamp,Node.id,Node.host,Node.metrics_port)\
+                                         .where(Node.status == UPGRADING)\
+                                         .order_by(Node.timestamp.asc())).all()
+        # Iterate through active upgrades
+        records_to_upgrade = len(upgrades)
+        for check in upgrades:
+            # If the DelayUpgrade timer has expired, check on status
+            if isinstance(check[0],int) and \
+                check[0] < (int(time.time()) - (config["DelayUpgrade"]*60)):
+                logging.info("Updating upgraded node "+str(check[1]))
+                node_metrics=read_node_metrics(check[2],check[3])
+                node_metadata=read_node_metadata(check[2],check[3])
+                if node_metrics and node_metadata:
+                    update_node_from_metrics(check[1],node_metrics,node_metadata)
+                records_to_upgrade-=1
+        old["UpgradingNodes"]=records_to_upgrade
+    # Are we already restarting a node
+    if old["RestartingNodes"]:
+        with S() as session:
+            restarts=session.execute(select(Node.timestamp,Node.id,Node.host,Node.metrics_port)\
+                                         .where(Node.status == RESTARTING)\
+                                         .order_by(Node.timestamp.asc())).all()
+        # Iterate through active upgrades
+        records_to_restart = len(restarts)
+        for check in restarts:
+            # If the DelayUpgrade timer has expired, check on status
+            if isinstance(check[0],int) and \
+                check[0] < (int(time.time()) - (config["DelayStart"]*60)):
+                logging.info("Updating restarted node "+str(check[1]))
+                node_metrics=read_node_metrics(check[2],check[3])
+                node_metadata=read_node_metadata(check[2],check[3])
+                if node_metrics and node_metadata:
+                    update_node_from_metrics(check[1],node_metrics,node_metadata)
+                records_to_restart-=1
+        old["RestartingNodes"]=records_to_restart
+    return(old)
+
+# Enable firewall for port
+def enable_firewall(port,node):
+    logging.info("enable firewall port {0}/udp".format(port))
+    # Close ufw firewall
+    try:
+        subprocess.run(['sudo','ufw','allow',"{0}/udp".format(port),'comment',node], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'EF Error:', err )
+
+# Disable firewall for port
+def disable_firewall(port):
+    logging.info("disable firewall port {0}/udp".format(port))
+    # Close ufw firewall
+    try:
+        subprocess.run(['sudo','ufw','delete','allow',"{0}/udp".format(port)], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'DF ERROR:', err )
+
+# Start a systemd node
+def start_systemd_node(node):
+    logging.info("Starting node "+str(node.id))
+    # Try to start the service
+    try:
+        p = subprocess.run(['sudo', 'systemctl', 'start', node.service], stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.decode('utf-8')
+        if re.match(r'Failed to start',p):
+            logging.error( 'SSN2 ERROR:', p )
+            return False
+    except subprocess.CalledProcessError as err:
+            logging.error( 'SSN1 ERROR:', err )
+            return False
+    # Open a firewall hole for the data port
+    enable_firewall(node.port,node.service)
+    # Update node status
+    set_node_status(node.id,RESTARTING)
+    return True
+
+# Stop a systemd node
+def stop_systemd_node(node):
+    logging.info("Stopping node: "+node.service)
+    # Send a stop signal to the process
+    try:
+        subprocess.run(['sudo', 'systemctl', 'stop', node.service], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'SSN2 ERROR:', err )
+    disable_firewall(node.port)
+    set_node_status(node.id,STOPPED)
+
+    return True
+
+# Upgrade a node
+def upgrade_node(node,metrics):
+    logging.info("Upgrading node "+str(node.id))
+    # Copy current node binary
+    try:
+        subprocess.run(['sudo', 'cp', '-f', metrics["antnode"], node.binary])
+    except subprocess.CalledProcessError as err:
+        logging.error( 'UN1 ERROR:', err )
+    try:
+        subprocess.run(['sudo', 'systemctl', 'restart', node.service])
+    except subprocess.CalledProcessError as err:
+        logging.error( 'UN2 ERROR:', err )
+    version=get_antnode_version(node.binary)
+    try:
+        with S() as session:
+            session.query(Node).filter(Node.id == node.id).\
+                update({'status': UPGRADING, 'timestamp': int(time.time()),
+                        'version': metrics["AntNodeVersion"]})
+            session.commit()
+    except:
+        return False
+    else:
+        return True
+
+# Remove a node
+def remove_node(id):
+    logging.info("Removing node "+str(id))
+
+    with S() as session:
+        node = session.execute(select(Node).where(Node.id == id)).first()
+    # Grab Node from Row
+    node=node[0]
+    if stop_systemd_node(node):
+        # Mark this node as REMOVING
+        set_node_status(id,REMOVING)
+
+        nodename=f"antnode{node.nodename}"
+        # Remove node data and log
+        try:
+            subprocess.run(['sudo', 'rm', '-rf', node.root_dir, f"/var/log/antnode/{nodename}"])
+        except subprocess.CalledProcessError as err:
+            logging.error( 'RN1 ERROR:', err )
+        # Remove systemd service file
+        try:
+            subprocess.run(['sudo', 'rm', '-f', f"/etc/systemd/system/{node.service}"])
+        except subprocess.CalledProcessError as err:
+            logging.error( 'RN2 ERROR:', err )
+        # Tell system to reload systemd files   
+        try:
+            subprocess.run(['sudo', 'systemctl', 'daemon-reload'])
+        except subprocess.CalledProcessError as err:
+            logging.error( 'RN3 ERROR:', err ) 
+    #print(json.dumps(node,indent=2))
+
+# Rescan nodes for status
+def update_nodes():
+    with S() as session:
+        nodes=session.execute(select(Node.timestamp,Node.id,Node.host,Node.metrics_port,Node.status)\
+                                        .where(Node.status != DISABLED)\
+                                        .order_by(Node.timestamp.asc())).all()
+    # Iterate through all records
+    for check in nodes:
+        # Check on status
+        if isinstance(check[0],int):
+            logging.debug("Updating info on node "+str(check[1]))
+            node_metrics=read_node_metrics(check[2],check[3])
+            node_metadata=read_node_metadata(check[2],check[3])
+            if node_metrics and node_metadata:
+                # Don't write updates for stopped nodes that are already marked as stopped
+                if node_metadata["status"]==STOPPED and check[4]==STOPPED:
+                    continue
+                update_node_from_metrics(check[1],node_metrics,node_metadata)
+     
+# Create a new node
+def create_node(config,metrics):
+    logging.info("Creating new node")
+    # Create a holding place for the new node
+    card = {}
+    # Find the next available node number by first looking for holes
+    sql = text('select n1.id + 1 as id from node n1 ' + \
+                'left join node n2 on n2.id = n1.id + 1 ' + \
+                'where n2.id is null ' + \
+                'and n1.id <> (select max(id) from node) ' + \
+                'order by n1.id;')
+    with S() as session:
+        result = session.execute(sql).first()
+    if result:
+        card['id']=result[0]
+    # Otherwise get the max node number and add 1
+    else:
+        with S() as session:
+            result = session.execute(select(Node.id).order_by(Node.id.desc())).first()
+        card['id']=result[0]+1
+    # Set the node name
+    card['nodename']=f'{card['id']:04}'
+    card['service']=f'antnode{card['nodename']}.service'
+    card['user']='ant'
+    card['version']=metrics["AntNodeVersion"]
+    card['root_dir']=f"{config['NodeStorage']}/antnode{card['nodename']}"
+    card['binary']=f"{card['root_dir']}/antnode"
+    card['port']=config["PortStart"]*1000+card['id']
+    card['metrics_port']=13*1000+card['id']
+    card['network']='evm-arbitrum-one'
+    card['wallet']=config["RewardsAddress"]
+    card['peer_id']=''
+    card['status']=STOPPED
+    card['timestamp']=int(time.time())
+    card['records']=0
+    card['uptime']=0
+    card['shunned']=0
+    card['age']=card['timestamp']
+    card['host']=ANM_HOST
+    log_dir=f"/var/log/antnode/antnode{card['nodename']}"
+    # Create the node directory and log directory
+    try:
+        subprocess.run(['sudo','mkdir','-p',card["root_dir"],log_dir], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'CN1 ERROR:', err )
+    # Copy the binary to the node directory
+    try:
+        subprocess.run(['sudo','cp',metrics["antnode"],card["root_dir"]], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'CN2 ERROR:', err )
+    # Change owner of the node directory and log directories
+    try:
+        subprocess.run(['sudo','chown','-R',f'{card["user"]}:{card["user"]}',card["root_dir"],log_dir], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'CN3 ERROR:', err )
+    # build the systemd service unit
+    service=f"""[Unit]
+Description=antnode{card['nodename']}
+[Service]
+User={card['user']}
+ExecStart={card['binary']} --bootstrap-cache-dir /var/antctl/bootstrap-cache --root-dir {card['root_dir']} --port {card['port']} --enable-metrics-server --metrics-server-port {card['metrics_port']} --log-output-dest {log_dir} --max-log-files 1 --max-archived-log-files 1 --rewards-address {card['wallet']} {card['network']}
+Restart=always
+#RestartSec=300
+"""
+    # Write the systemd service unit with sudo tee since we're running as not root
+    try:
+        subprocess.run(['sudo','tee',f'/etc/systemd/system/{card["service"]}'],input=service,text=True, stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'CN4 ERROR:', err )
+    # Reload systemd service files to get our new one
+    try:
+        subprocess.run(['sudo','systemctl','daemon-reload'], stdout=subprocess.PIPE)
+    except subprocess.CalledProcessError as err:
+            logging.error( 'CN5 ERROR:', err )
+    # Add the new node to the database
+    with S() as session:
+        session.execute(
+            insert(Node),[card]
+        )
+        session.commit()
+    # Now we grab the node object from the database to pass to start node
+    with S() as session:
+        card=session.execute(select(Node).where(Node.id == card['id'])).first()
+    # Get the Node object from the Row
+    card=card[0]
+    # Start the new node
+    return start_systemd_node(card)
+    #print(json.dumps(card,indent=2))
+    return True
+    
+
+# Make a decision about what to do
+def choose_action(config,metrics,db_nodes):
+    # Gather knowlege
+    features={}
+    features["AllowCpu"]=metrics["UsedCpuPercent"] < config["CpuLessThan"]
+    features["AllowMem"]=metrics["UsedMemPercent"] < config["MemLessThan"]
+    features["AllowHD"]=metrics["UsedHDPercent"] < config["HDLessThan"]
+    features["RemCpu"]=metrics["UsedCpuPercent"] > config["CpuRemove"]
+    features["RemMem"]=metrics["UsedMemPercent"] > config["MemRemove"]
+    features["RemHD"]=metrics["UsedHDPercent"] > config["HDRemove"]
+    features["AllowNodeCap"]=metrics["RunningNodes"] < config["NodeCap"]
+    # These are new features, so ignore them if not configured
+    if (config["NetIOReadLessThan"]+config["NetIOReadRemove"]+
+        config["NetIOWriteLessThan"]+config["NetIOWriteRemove"]>1):
+        features["AllowNetIO"]=metrics["NetReadBytes"] < config["NetIOReadLessThan"] and \
+                              metrics["NetWriteBytes"] < config["NetIOWriteLessThan"]
+        features["RemoveNetIO"]=metrics["NetReadBytes"] > config["NetIORemove"] or \
+                              metrics["NetWriteBytes"] > config["NetIORemove"]
+    else:
+        features["AllowNetIO"]=True
+        features["RemoveNetIO"]=False
+    if (config["HDIOReadLessThan"]+config["HDIOReadRemove"]+
+        config["HDIOWriteLessThan"]+config["HDIOWriteRemove"]>1):
+        features["AllowHDIO"]=metrics["HDReadBytes"] < config["HDIOReadLessThan"] and \
+                              metrics["HDWriteBytes"] < config["HDIOWriteLessThan"]
+        features["RemoveHDIO"]=metrics["HDReadBytes"] > config["HDIORemove"] or \
+                              metrics["HDWriteBytes"] > config["HDtIORemove"]
+    else:
+        features["AllowHDIO"]=True
+        features["RemoveHDIO"]=False
+    features["LoadAllow"] = metrics["LoadAverage1"] < config["DesiredLoadAverage"] and \
+                            metrics["LoadAverage5"] < config["DesiredLoadAverage"] and \
+                            metrics["LoadAverage15"] < config["DesiredLoadAverage"] 
+    features["LoadNotAllow"] = metrics["LoadAverage1"] > config["MaxLoadAverageAllowed"] or \
+                            metrics["LoadAverage5"] > config["MaxLoadAverageAllowed"] or \
+                            metrics["LoadAverage15"] > config["MaxLoadAverageAllowed"] 
+    # Check records for expired status
+    metrics=update_counters(metrics,config)
+    # If we have other thing going on, don't add more nodes
+    features["AddNewNode"]=sum([ metrics.get(m, 0) \
+                                for m in ['UpgradingNodes',
+                                          'RestartingNodes','MigratingNodes',
+                                          'RemovingNodes'] ]) == 0 and \
+                            features["AllowCpu"] and features["AllowHD"] and \
+                            features["AllowMem"] and features["AllowNodeCap"] and \
+                            features["AllowHDIO"] and features["AllowNetIO"] and \
+                            features["LoadAllow"]
+    # Are we overlimit on nodes
+    features["Remove"] =features["LoadNotAllow"] or features["RemCpu"] or \
+                        features["RemHD"] or features["RemMem"] or \
+                        features["RemoveHDIO"] or features["RemoveNetIO"] or \
+                        metrics["TotalNodes"] > config["NodeCap"]
+    # If we have nodes to upgrade
+    if metrics["NodesToUpgrade"] >= 1:
+        # Make sure current version is equal or newer than version on first node.
+        if Version(metrics["AntNodeVersion"]) < Version(db_nodes[0][1]):
+            logging.warning("node upgrade cancelled due to lower version")
+            features["Upgrade"]=False
+        else:
+            if features["Remove"]:
+                logging.info("Can't upgrade while removing is required")
+                features["Upgrade"]=False
+            else:
+                features["Upgrade"]=True
+    else:
+        features["Upgrade"]=False
+
+
+    logging.info(json.dumps(features,indent=2))
+    ##### Decisions
+
+    # Actually, removing DEAD nodes take priority
+    if metrics["DeadNodes"] > 1:
+        with S() as session:
+            broken=session.execute(select(Node.timestamp,Node.id,Node.host,Node.metrics_port)\
+                                         .where(Node.status == DEAD)\
+                                         .order_by(Node.timestamp.asc())).all()
+        # Iterate through dead nodes and remove them all
+        for check in broken:
+            # Remove broken nodes
+            logging.info("Removing dead node "+str(check[1]))
+            remove_node(check[1])
+        return {"status": "removed-dead-nodes"}
+    # If we have nodes with no version number, update from binary
+    if metrics["NodesNoVersion"] > 1:
+        with S() as session:
+            no_version=session.execute(select(Node.timestamp,Node.id,Node.binary)\
+                                         .where(Node.version == '')\
+                                         .order_by(Node.timestamp.asc())).all()
+        # Iterate through nodes with no version number
+        for check in no_version:
+            # Update version number from binary
+            version=get_antnode_version(check[2])
+            logging.info(f"Updating version number for node {check[1]} to {version}")
+            with S() as session:
+                session.query(Node).filter(Node.id == check[1]).\
+                    update({'version': version})
+                session.commit()
+            
+    # If we're restarting, wait patiently as metrics could be skewed
+    if metrics["RestartingNodes"]:
+        logging.info("Still waiting for RestartDelay")
+        return {"status": RESTARTING}
+    # If we still have unexpired upgrade records, wait
+    if metrics["UpgradingNodes"]:
+        logging.info("Still waiting for UpgradeDelay")
+        return {"status": UPGRADING}
+    # First if we're removing, that takes top priority
+    if features["Remove"]:
+        # If we still have unexpired removal records, wait
+        if metrics["RemovingNodes"]:
+            logging.info("Still waiting for RemoveDelay")
+            return {"status": REMOVING}
+        # If we're under HD pressure or trimming node cap, remove nodes
+        if features["RemHD"] or metrics["TotalNodes"] > config["NodeCap"]:
+            # Start removing with stopped nodes
+            if metrics["StoppedNodes"] > 0:
+                # What is the youngest stopped node
+                with S() as session:
+                    youngest=session.execute(select(Node.id)\
+                                            .where(Node.status == STOPPED)\
+                                            .order_by(Node.age.desc())).first()
+                if youngest:
+                    # Remove the youngest node
+                    remove_node(youngest[0])
+                    return{"status": REMOVING}
+            # No low hanging fruit. let's start with the youngest running node
+            with S() as session:
+                youngest=session.execute(select(Node.id)\
+                                        .where(Node.status == RUNNING)\
+                                        .order_by(Node.age.desc())).first()
+            if youngest:
+                # Remove the youngest node
+                remove_node(youngest[0])
+                return{"status": REMOVING}
+            return{"status": "nothing-to-remove"}
+        # Otherwise, let's try just stopping a node to bring IO/Mem/Cpu down
+        else:
+            # If we just stopped a node, wait
+            if int(config["LastStoppedAt"] or 0) > (int(time.time()) - (config["DelayRemove"]*60)):
+                logging.info("Still waiting for RemoveDelay")
+                return {"status": 'waiting-to-stop'}
+            # Start with the youngest running node
+            with S() as session:
+                youngest=session.execute(select(Node)\
+                                        .where(Node.status == RUNNING)\
+                                        .order_by(Node.age.desc())).first()
+            if youngest:
+                # Stop the youngest node
+                stop_systemd_node(youngest[0])
+                # Update the last stopped time
+                with S() as session:
+                    session.query(Machine).filter(Machine.id == 1).\
+                        update({'LastStoppedAt': int(time.time())})
+                    session.commit()
+                return{"status": STOPPED}
+            else:
+                return{"status": "nothing-to-stop"}
+    
+    # Do we have upgrading to do?
+    if features["Upgrade"]:    
+        # Let's find the oldest running node not using the current version
+        with S() as session:
+            oldest=session.execute(select(Node)\
+                                        .where(Node.status == RUNNING)\
+                                        .where(Node.version != metrics["AntNodeVersion"])
+                                        .order_by(Node.age.asc())).first()
+        if oldest:
+            # Get Node from Row
+            oldest = oldest[0]
+            # If we don't have a version number from metadata, grab from binary
+            if not oldest.version:
+                oldest.version=get_antnode_version(oldest.binary)
+            #print(json.dumps(oldest))
+            # Upgrade the oldest node
+            upgrade_node(oldest,metrics)
+            return{"status": UPGRADING}
+        
+    # If AddNewNode
+    #   If stopped nodes available
+    #     Check oldest stopped version
+    #     If out of date
+    #         upgrade node which starts it
+    #     else
+    #         restart node
+    #   else
+    #     Create a Node which starts it
+    if features["AddNewNode"]:
+        # Start adding with stopped nodes
+        if metrics["StoppedNodes"] > 0:
+            # What is the oldest stopped node
+            with S() as session:
+                oldest=session.execute(select(Node)\
+                                        .where(Node.status == STOPPED)\
+                                        .order_by(Node.age.asc())).first()
+            if oldest:
+                # Get Node from Row
+                oldest=oldest[0]
+                # If we don't have a version number from metadata, grab from binary
+                if not oldest.version:
+                    oldest.version=get_antnode_version(oldest.binary)
+                # If the stopped version is old, upgrade it
+                if Version(metrics["AntNodeVersion"]) > Version(oldest.version):
+                    upgrade_node(oldest,metrics)
+                    return{"status": UPGRADING}
+                else:
+                    if start_systemd_node(oldest):
+                        return{"status": RESTARTING}
+                    else:
+                        return{"status": "failed-start-node"}
+            # Hmm, still in Start mode, we shouldn't get here
+            return {"status": 'START'}
+        # Still in Add mode, add a new node
+        if metrics["TotalNodes"] < config["NodeCap"]:
+            if create_node(config,metrics):
+                return {"status": "ADD"}
+            else:
+                return {"status": "failed-create-node"}
+        else:
+            return {"status": "node-cap-reached"}
+    # If we have nothing to do, Survey the node ports
+    update_nodes()
+    return{"status": "idle"}
+
+def main():
+    # We're starting, so lets create a lock file
+    try:
+        with open('/var/antctl/wnm_active', 'w') as file:
+            file.write(str(int(time.time())))
+    except:
+        logging.error("Unable to create lock file, exiting")
+        sys.exit(1)
+
+    # See if we already have a known state in the database
+    with S() as session:
+        db_nodes=session.execute(select(Node.status,Node.version,
+                                        Node.host,Node.metrics_port,
+                                        Node.port,Node.age,Node.id,
+                                        Node.timestamp)).all()
+        anm_config=session.execute(select(Machine)).all()
+
+    if db_nodes:
+        # anm_config by default loads a parameter array, 
+        # use the __json__ method to return a dict from the first node
+        anm_config = json.loads(json.dumps(anm_config[0][0])) or load_anm_config()
+        metrics=get_machine_metrics(anm_config["NodeStorage"],anm_config["HDRemove"])
+        #node_metrics = read_node_metrics(db_nodes[0][2],db_nodes[0][3])
+        #print(db_nodes[0])
+        #print(node_metrics)
+        #print(anm_config)
+        #print(json.dumps(anm_config,indent=4))
+        #print("Node: ",db_nodes)
+        logging.info("Found {counter} nodes migrated".format(counter=len(db_nodes)))
+
+    else:
+        anm_config = load_anm_config()
+        #print(anm_config)
+        Workers = survey_machine() or []
+
+        #""""
+        with S() as session:
+            session.execute(
+                insert(Node),Workers
+            )
+            session.commit()
+        #"""
+
+        with S() as session:
+            session.execute(
+                insert(Machine),[anm_config]
+            )
+            session.commit()
+
+        # Now load subset of data to work with
+        with S() as session:
+            db_nodes=session.execute(select(Node.status,Node.version,
+                                            Node.host,Node.metrics_port,
+                                            Node.port,Node.age,Node.id,
+                                            Node.timestamp)).all()
+
+
+
+        #print(json.dumps(anm_config,indent=4))
+        logging.info("Found {counter} nodes configured".format(counter=len(db_nodes)))
+
+        #versions = [v[1] for worker in Workers if (v := worker.get('version'))]
+        #data = Counter(ver for ver in versions)
+
+
+    data = Counter(status[0] for status in db_nodes)
+    #print(data)
+    print("Running Nodes:",data[RUNNING])
+    print("Restarting Nodes:",data[RESTARTING])
+    print("Stopped Nodes:",data[STOPPED])
+    print("Upgrading Nodes:",data[UPGRADING])
+    print("Removing Nodes:",data[REMOVING])
+    data = Counter(ver[1] for ver in db_nodes)
+    print("Versions:",data)
+
+    machine_metrics = get_machine_metrics(anm_config['NodeStorage'],anm_config["HDRemove"])
+    print(json.dumps(anm_config,indent=2))
+    print(json.dumps(machine_metrics,indent=2))
+    this_action=choose_action(anm_config,machine_metrics,db_nodes)
+    print("Action:",json.dumps(this_action,indent=2))
+    # Remove lock file
+    os.remove("/var/antctl/wnm_active")
+
+if __name__ == "__main__":
+    main()
+
+print("End of program")

wnm/config.py

@@ -0,0 +1,3 @@
+import os, sys
+from dotenv import load_dotenv
+

wnm/models.py

@@ -0,0 +1,192 @@
+import json_fix
+
+# Turn a class into a storable object with ORM
+from typing import Optional
+from sqlalchemy import Integer, Unicode, UnicodeText, Float
+from sqlalchemy import create_engine, select, insert, update
+from sqlalchemy.orm import sessionmaker, scoped_session
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.orm import Mapped, mapped_column
+
+# create a Base class bound to sqlalchemy
+class Base(DeclarativeBase):
+    pass
+
+# Extend the Base class to create our Host info
+class Machine(Base):
+    __tablename__ = 'machine'
+    # No schema in sqlite3
+    # __table_args__ = {"schema": "colony"}
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    CpuCount: Mapped[int] = mapped_column(Integer)
+    NodeCap: Mapped[int] = mapped_column(Integer)
+    CpuLessThan: Mapped[int] = mapped_column(Integer)
+    CpuRemove: Mapped[int] = mapped_column(Integer)
+    MemLessThan: Mapped[int] = mapped_column(Integer)
+    MemRemove: Mapped[int] = mapped_column(Integer)
+    HDLessThan: Mapped[int] = mapped_column(Integer)
+    HDRemove: Mapped[int] = mapped_column(Integer)
+    DelayStart: Mapped[int] = mapped_column(Integer)
+    DelayUpgrade: Mapped[int] = mapped_column(Integer)
+    DelayRemove: Mapped[int] = mapped_column(Integer)
+    NodeStorage: Mapped[str] = mapped_column(UnicodeText)
+    RewardsAddress: Mapped[str] = mapped_column(UnicodeText)
+    DonateAddress: Mapped[str] = mapped_column(UnicodeText)
+    MaxLoadAverageAllowed: Mapped[float] = mapped_column(Float)
+    DesiredLoadAverage: Mapped[float] = mapped_column(Float)
+    # What port to begin assigning nodes
+    PortStart: Mapped[int] = mapped_column(Integer)
+    HDIOReadLessThan: Mapped[float] = mapped_column(Float)
+    HDIOReadRemove: Mapped[float] = mapped_column(Float)
+    HDIOWriteLessThan: Mapped[float] = mapped_column(Float)
+    HDIOWriteRemove: Mapped[float] = mapped_column(Float)
+    NetIOReadLessThan: Mapped[float] = mapped_column(Float)
+    NetIOReadRemove: Mapped[float] = mapped_column(Float)
+    NetIOWriteLessThan: Mapped[float] = mapped_column(Float)
+    NetIOWriteRemove: Mapped[float] = mapped_column(Float)
+    LastStoppedAt: Mapped[int] = mapped_column(Integer)
+
+    def __init__(self, CpuCount, NodeCap, CpuLessThan, CpuRemove,
+                 MemLessThan, MemRemove, HDLessThan, HDRemove,
+                 DelayStart, DelayUpgrade, DelayRemove, NodeStorage,
+                 RewardsAddress, DonateAddress, MaxLoadAverageAllowed, 
+                 DesiredLoadAverage, PortStart, HDIOReadLessThan, 
+                 HDIOReadRemove, HDIOWriteLessThan, HDIOWriteRemove, 
+                 NetIOReadLessThan, NetIOReadRemove, NetIOWriteLessThan, 
+                 NetIOWriteRemove, LastStoppedAt):
+        
+        self.CpuCount = CpuCount
+        self.NodeCap = NodeCap
+        self.CpuLessThan = CpuLessThan
+        self.CpuRemove = CpuRemove
+        self.MemLessThan = MemLessThan
+        self.MemRemove = MemRemove
+        self.HDLessThan = HDLessThan
+        self.HDRemove = HDRemove
+        self.DelayStart = DelayStart
+        self.DelayUpgrade = DelayUpgrade
+        self.DelayRemove = DelayRemove
+        self.NodeStorage = NodeStorage
+        self.RewardsAddress = RewardsAddress
+        self.DonateAddress = DonateAddress
+        self.MaxLoadAverageAllowed = MaxLoadAverageAllowed
+        self.DesiredLoadAverage = DesiredLoadAverage
+        self.PortStart = PortStart
+        self.HDIOReadLessThan = HDIOReadLessThan
+        self.HDIOReadRemove = HDIOReadRemove
+        self.HDIOWriteLessThan = HDIOWriteLessThan
+        self.HDIOWriteRemove = HDIOWriteRemove
+        self.NetIOReadLessThan = NetIOReadLessThan
+        self.NetIOReadRemove = NetIOReadRemove
+        self.NetIOWriteLessThan = NetIOWriteLessThan
+        self.NetIOWriteRemove = NetIOWriteRemove
+        self.LastStoppedAt = LastStoppedAt
+
+    def __repr__(self):
+        return f'Machine({self.CpuCount},{self.NodeCap},{self.CpuLessThan},{self.CpuRemove}' + \
+            f',{self.MemLessThan},{self.MemRemove},{self.HDLessThan}' + \
+            f',{self.HDRemove},{self.DelayStart},{self.DelayUpgrade}' + \
+            f',{self.DelayRemove}' + \
+            f',"{self.NodeStorage}","{self.RewardsAddress}","{self.DonateAddress}"' + \
+            f',{self.MaxLoadAverageAllowed},{self.DesiredLoadAverage}' + \
+            f',{self.PortStart},{self.HDIOReadLessThan},{self.HDIOReadRemove}' + \
+            f',{self.HDIOWriteLessThan},{self.HDIOWriteRemove}' + \
+            f',{self.NetIOReadLessThan},{self.NetIOReadRemove}' + \
+            f',{self.NetIOWriteLessThan},{self.NetIOWriteRemove}' + \
+            f',{self.LastStoppedAt})'
+    
+    def __json__(self):
+        return { 'CpuCount': self.CpuCount, 'NodeCap': self.NodeCap, 'CpuLessThan': self.CpuLessThan,
+            'CpuRemove': self.CpuRemove, 'MemLessThan': self.MemLessThan, 'MemRemove': self.MemRemove,
+            'HDLessThan': self.HDLessThan, 'HDRemove': self.HDRemove, 'DelayStart': self.DelayStart,
+            'DelayUpgrade': self.DelayUpgrade, 'DelayRemove': self.DelayRemove, 'NodeStorage': f"{self.NodeStorage}",
+            'RewardsAddress': f"{self.RewardsAddress}", 'DonateAddress': f"{self.DonateAddress}",
+            'MaxLoadAverageAllowed': self.MaxLoadAverageAllowed, 'DesiredLoadAverage': self.DesiredLoadAverage,
+            'PortStart': self.PortStart, 'HDIOReadLessThan': self.HDIOReadLessThan, 'HDIOReadRemove': self.HDIOReadRemove,
+            'HDIOWriteLessThan': self.HDIOWriteLessThan, 'HDIOWriteRemove': self.HDIOWriteRemove,
+            'NetIOReadLessThan': self.NetIOReadLessThan, 'NetIOReadRemove': self.NetIOReadRemove,
+            'NetIOWriteLessThan': self.NetIOWriteLessThan, 'NetIOWriteRemove': self.NetIOWriteRemove,
+            'LastStoppedAt': self.LastStoppedAt }
+
+   
+# Extend the Base class to create our Node info
+class Node(Base):
+    __tablename__ = 'node'
+    # No schema in sqlite3
+    #__table_args__ = {"schema": "colony"}
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    # Maps to antnode-{nodename}
+    nodename: Mapped[str] = mapped_column(Unicode(10))
+    # service definition name
+    service: Mapped[str] = mapped_column(UnicodeText)
+    # User running node
+    user: Mapped[str] = mapped_column(Unicode(24))
+    # Full path to node binary
+    binary: Mapped[str] = mapped_column(UnicodeText)
+    # Last polled version of the binary
+    version: Mapped[Optional[str]] = mapped_column(UnicodeText)
+    # Root directory of the node
+    root_dir: Mapped[str] = mapped_column(UnicodeText)
+    # Node open port
+    port: Mapped[int] = mapped_column(Integer)
+    # Node metrics port
+    metrics_port: Mapped[int] = mapped_column(Integer)
+    # Network to use ( Live is evm-arbitrum-one )
+    network: Mapped[str] = mapped_column(UnicodeText)
+    # Reward address
+    wallet: Mapped[Optional[str]] = mapped_column(Unicode(42),index=True)
+    # Reported peer_id
+    peer_id: Mapped[Optional[str]] = mapped_column(Unicode(52))
+    # Node's last probed status
+    status: Mapped[str] = mapped_column(Unicode(32),index=True)
+    # Timestamp of last update
+    timestamp: Mapped[int] = mapped_column(Integer,index=True)
+    # Number of node records stored as reported by node
+    records: Mapped[int] = mapped_column(Integer,index=True)
+    # Node reported uptime
+    uptime: Mapped[int] = mapped_column(Integer)
+    # Number of shuns
+    shunned: Mapped[int] = mapped_column(Integer)
+    # Timestamp of node first launch
+    age: Mapped[int] = mapped_column(Integer)
+    # Host ip/name for data and metrics ports
+    host: Mapped[Optional[str]] = mapped_column(UnicodeText)
+
+    def __init__(self, id, nodename, service, user, binary, version, 
+                 root_dir, port, metrics_port, network,
+                 wallet, peer_id, status, timestamp, records,
+                 uptime, shunned, age, host):
+        self.id = id
+        self.nodename = nodename
+        self.service = service
+        self.user = user
+        self.binary = binary
+        self.version = version
+        self.root_dir = root_dir
+        self.port = port
+        self.metrics_port = metrics_port
+        self.network = network
+        self.wallet = wallet
+        self.peer_id = peer_id
+        self.status = status
+        self.timestamp = timestamp
+        self.records = records
+        self.uptime = uptime
+        self.shunned = shunned
+        self.age = age
+        self.host = host
+
+    def __repr__(self):
+        return f'Node({self.id},"{self.nodename}","{self.service}","{self.user},"{self.binary}"'+\
+            f',"{self.version}","{self.root_dir}",{self.port},{self.metrics_port}' + \
+            f',"{self.network}","{self.wallet}","{self.peer_id}","{self.status}",{self.timestamp}' + \
+            f',{self.records},{self.uptime},{self.shunned},{self.age},"{self.host}")'
+    
+    def __json__(self):
+        return { "id": self.id, "nodename": f"{self.nodename}", "service": f"{self.service}",
+            "user": f"{self.user}", "binary": f"{self.binary}", "version": f"{self.version}",
+            "root_dir": f"{self.root_dir}", "port": self.port, "metrics_port": self.metrics_port,
+            "network": f"{self.network}", "wallet": f"{self.wallet}", "peer_id": f"{self.peer_id}",
+            "status": f"{self.status}", "timestamp": self.timestamp, "records": self.records,
+            "uptime": self.uptime, "shunned": self.shunned, "age": self.age,
+            "host": f"{self.host}" }

wnm-0.0.4.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: wnm
+Version: 0.0.4
+Summary: Manager for Autonomi nodes
+Author-email: Troy Johnson <troy@weave.sh>
+License: GPL-3.0
+Keywords: Autonomi,antnode,weave,xd7
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.12.3
+Description-Content-Type: text/markdown
+Requires-Dist: requests
+Requires-Dist: packaging
+Requires-Dist: sqlalchemy
+Requires-Dist: alembic
+Requires-Dist: json-fix
+Requires-Dist: psutil
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: isort; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+
+# Weave Node Manager
+
+## Overview
+Weave Node Manager (wnm) is a Python application designed to manage nodes for decentralized networks.
+
+## Features
+- Update node metrics and statuses.
+- Manage systemd services and ufw firewall for linux nodes.
+- Support for configuration via YAML, JSON, or command-line parameters.
+
+## Installation
+1. Clone the repository:
+   ```
+   git clone https://github.com/iweave/weave-node-manager.git
+   ```
+2. Navigate to the project directory:
+   ```
+   cd weave-node-manager
+3. Create a virtual environment
+   ```
+   python3 -m venv .venv
+   ```
+4. Activate the virtual environment
+   ```
+   . .venv/bin/activate
+   ```
+5. Install the required dependencies:
+   ```
+   pip install -r requirements.txt
+   ```
+
+## Configuration
+Configuration can be done through a `.env` file, YAML, or JSON files. The application will prioritize these configurations over default values.
+
+Upon finding an existing installation of [anm - aatonnomicc node manager](https://github.com/safenetforum-community/NTracking/tree/main/anm), wnm will disable anm and take over management of the cluster. The /var/antctl/config is only read on first ingestion, configuration priority then moves to the `.env` file or a named configuration file.
+
+## Usage
+To run the application, execute the following command:
+```
+python main.py
+```
+
+## Contributing
+Contributions are welcome! Please submit a pull request or open an issue for any enhancements or bug fixes.
+
+## License
+This project is licensed under the MIT License. See the LICENSE file for more details.

wnm-0.0.4.dist-info/RECORD

@@ -0,0 +1,9 @@
+wnm/__init__.py,sha256=3WMpwTP3eL5shixA3OMks-dTlTUzqN6pge2F4DQxEFo,91
+wnm/__main__.py,sha256=IXboB7V4o80ye6N7xNAteoD_CALa4m49RlNduNLcifY,42631
+wnm/config.py,sha256=nMJlzxZX7NA0iI_QAuJDE0uI3V_ROjP5v99wdKDfuDs,47
+wnm/models.py,sha256=7lhZkUfFlnDgeEVRngQVuFnzSbJPwbytLMCynAo1j28,9416
+wnm-0.0.4.dist-info/METADATA,sha256=1UvyYwEc5OyyiVU3_jq2lZZw5mqRB-GwL8VcDfDt44Y,2539
+wnm-0.0.4.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+wnm-0.0.4.dist-info/entry_points.txt,sha256=jfoemjoLVPeeiBMHKqAExrHQ4Rhf9IXxL4JCnS7ZYFo,42
+wnm-0.0.4.dist-info/top_level.txt,sha256=E6dTE5k6efMEB9LaJAZSBu8zzs__l4R55t0-F-LwufI,4
+wnm-0.0.4.dist-info/RECORD,,

wnm-0.0.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (77.0.3)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

wnm-0.0.4.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+wnm = wnm.__main__:main

wnm-0.0.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+wnm