plc-checkweigher 1.32.2 → 1.33.0

package/bin/plc_checkweigher

@@ -174,12 +174,13 @@ restart)
     need_sudo
     banner "Restarting services"
     echo ""
-    spin_start "Restarting plc_watcher and plc_web"
-    sudo systemctl restart plc_watcher plc_web
+    spin_start "Restarting plc_watcher, plc_web and plc_selfheal"
+    sudo systemctl restart plc_watcher plc_web plc_selfheal 2>/dev/null \
+        || sudo systemctl restart plc_watcher plc_web
     sleep 2
     spin_ok
     echo ""
-    systemctl status plc_watcher plc_web --no-pager | grep -E 'Active|Main PID'
+    systemctl status plc_watcher plc_web plc_selfheal --no-pager 2>/dev/null | grep -E 'Active|Main PID'
     echo ""
     ;;
 
@@ -187,8 +188,9 @@ start)
     need_sudo
     banner "Starting services"
     echo ""
-    spin_start "Starting plc_watcher and plc_web"
-    sudo systemctl start plc_watcher plc_web
+    spin_start "Starting plc_watcher, plc_web and plc_selfheal"
+    sudo systemctl start plc_watcher plc_web plc_selfheal 2>/dev/null \
+        || sudo systemctl start plc_watcher plc_web
     sleep 1
     spin_ok
     echo ""
@@ -198,12 +200,15 @@ stop)
     need_sudo
     banner "Stopping services"
     echo ""
-    spin_start "Stopping plc_watcher and plc_web"
+    # Stop the self-healing daemon FIRST — otherwise it would detect the
+    # watcher/web going down and immediately restart them, fighting this stop.
+    spin_start "Stopping plc_selfheal, plc_watcher and plc_web"
+    sudo systemctl stop plc_selfheal 2>/dev/null || true
     sudo systemctl stop plc_watcher plc_web
     spin_ok
     echo ""
     warn "Services stopped — will restart automatically on next boot"
-    warn "To disable auto-start: sudo systemctl disable plc_watcher plc_web"
+    warn "To disable auto-start: sudo systemctl disable plc_watcher plc_web plc_selfheal"
     echo ""
     ;;
 
@@ -235,6 +240,87 @@ queue)
     echo ""
     ;;
 
+# ── Self-healing daemon ────────────────────────────────────────────────────────
+selfheal)
+    SUBCMD="${1:-status}"
+    shift || true
+    case "$SUBCMD" in
+    status)
+        banner "Self-Healing Daemon"
+        echo ""
+        ACTIVE=$(systemctl is-active   plc_selfheal 2>/dev/null || true)
+        ENABLED=$(systemctl is-enabled plc_selfheal 2>/dev/null || true)
+        if [[ "$ACTIVE" == "active" ]]; then
+            ok "plc_selfheal: RUNNING  (auto-start: ${ENABLED})"
+        else
+            warn "plc_selfheal: ${ACTIVE}  (auto-start: ${ENABLED})"
+            info "Start it:  sudo systemctl start plc_selfheal"
+        fi
+        echo ""
+        # Last-cycle / throttle state
+        _ST="/home/pi/reports/health/.selfheal_state.json"
+        if [[ -f "$_ST" ]]; then
+            "${PYTHON}" - "$_ST" << 'PYEOF' 2>/dev/null || true
+import json, sys, time
+try:
+    d = json.load(open(sys.argv[1]))
+except Exception:
+    sys.exit(0)
+lc = d.get("last_cycle", 0)
+if lc:
+    age = int(time.time() - lc)
+    print(f"  Last self-heal sweep: {age}s ago")
+rep = d.get("reported", {})
+if rep:
+    print(f"  Active unresolved problems: {len(rep)}")
+    for k in rep: print(f"     • {k}")
+else:
+    print("  No unresolved problems")
+PYEOF
+        else
+            info "No sweep has run yet (daemon may have just started)"
+        fi
+        echo ""
+        # Pending health reports on disk
+        _HD="/home/pi/reports/health"
+        if [[ -d "$_HD" ]]; then
+            _N=$(find "$_HD" -maxdepth 1 -name 'health_*.txt' 2>/dev/null | wc -l)
+            [[ "${_N:-0}" -gt 0 ]] && info "${_N} health report(s) in ${_HD}"
+        fi
+        echo ""
+        info "Recent activity:  plc_checkweigher selfheal logs"
+        echo ""
+        ;;
+    logs)
+        banner "Self-Heal Logs"
+        echo ""
+        exec journalctl -u plc_selfheal -n 50 -f --no-pager
+        ;;
+    now|run)
+        banner "Self-Heal — Immediate Sweep"
+        need_sudo
+        echo ""
+        spin_start "Running one self-heal cycle"
+        sudo "${PYTHON}" -c "
+import sys; sys.path.insert(0, '${INSTALL_DIR}')
+import selfheal
+healed, failed, env = selfheal.run_cycle()
+selfheal.maybe_report(healed, failed, env)
+print('HEALED:', len(healed), '| UNRESOLVED:', len(failed))
+for k,d in healed: print('  [HEALED]', k, '-', d)
+for k,d in failed: print('  [FAIL]  ', k, '-', d)
+" && spin_ok || spin_warn "Cycle completed with issues"
+        echo ""
+        ;;
+    *)
+        echo "Usage: plc_checkweigher selfheal [status|logs|now]"
+        echo "  status  — daemon state, last sweep, unresolved problems (default)"
+        echo "  logs    — follow the self-heal journal"
+        echo "  now     — run one self-heal sweep immediately"
+        ;;
+    esac
+    ;;
+
 # ── Push test ─────────────────────────────────────────────────────────────────
 push-test)
     banner "SMB Push Test"
@@ -1088,18 +1174,56 @@ SMBC
             echo ""
         fi
 
-        # ── 8. plc_live.json staleness ────────────────────────────────────────
+        # ── 8. Live state file (/tmp/plc_live.json) — service-aware repair ─────
+        # The watcher writes this file continuously. If it is missing or stale
+        # we decide the fix by the service state, not by the file alone:
+        #   • watcher inactive          → start it
+        #   • watcher active + missing  → stuck before first write → restart
+        #   • watcher active + stale    → lost PLC / stuck loop     → restart
+        # After any restart we wait briefly and confirm the file reappears.
         spin_start "Live state file (/tmp/plc_live.json)"
-        if [[ -f "/tmp/plc_live.json" ]]; then
-            _AGE=$(( $(date +%s) - $(stat -c %Y /tmp/plc_live.json 2>/dev/null || echo 0) ))
-            if [[ "${_AGE:-0}" -gt 10 ]]; then
-                ffix_warn "plc_live.json is ${_AGE}s old — watcher may be stuck or offline"
-                flog "WARN: plc_live.json stale (${_AGE}s old)"
+        _LIVE="/tmp/plc_live.json"
+        _WATCHER_ACTIVE=0
+        systemctl is-active --quiet plc_watcher 2>/dev/null && _WATCHER_ACTIVE=1
+
+        _LIVE_PROBLEM=""   # empty = healthy
+        if [[ -f "$_LIVE" ]]; then
+            _AGE=$(( $(date +%s) - $(stat -c %Y "$_LIVE" 2>/dev/null || echo 0) ))
+            [[ "${_AGE:-0}" -gt 10 ]] && _LIVE_PROBLEM="stale (${_AGE}s old)"
+        else
+            _LIVE_PROBLEM="missing"
+        fi
+
+        if [[ -z "$_LIVE_PROBLEM" ]]; then
+            ffix_info "plc_live.json updated ${_AGE}s ago  (OK)"
+        elif [[ $_WATCHER_ACTIVE -eq 0 ]]; then
+            ffix_warn "plc_live.json ${_LIVE_PROBLEM} — plc_watcher is not running"
+            flog "WARN: live state ${_LIVE_PROBLEM}; plc_watcher inactive"
+            spin_start "Starting plc_watcher"
+            sudo systemctl start plc_watcher 2>/dev/null || true
+            sleep 4
+            if [[ -f "$_LIVE" ]]; then
+                ffix_ok "plc_watcher started — live state now being written"
             else
-                ffix_info "plc_live.json updated ${_AGE}s ago  (OK)"
+                ffix_err "plc_watcher started but live state still missing — check: journalctl -u plc_watcher"
             fi
         else
-            ffix_info "plc_live.json absent  (normal — created when watcher connects)"
+            # Service is up but the file is missing/stale → watcher stuck. Restart.
+            ffix_warn "plc_live.json ${_LIVE_PROBLEM} while plc_watcher is running — restarting watcher"
+            flog "WARN: live state ${_LIVE_PROBLEM}; watcher active → restart"
+            spin_start "Restarting plc_watcher"
+            sudo systemctl restart plc_watcher 2>/dev/null || true
+            sleep 4
+            if [[ -f "$_LIVE" ]]; then
+                _AGE2=$(( $(date +%s) - $(stat -c %Y "$_LIVE" 2>/dev/null || echo 0) ))
+                if [[ "${_AGE2:-99}" -le 10 ]]; then
+                    ffix_ok "Watcher restarted — live state fresh (${_AGE2}s old)"
+                else
+                    ffix_warn "Watcher restarted but live state still stale — PLC may be unreachable"
+                fi
+            else
+                ffix_err "Watcher restarted but live state still missing — check: journalctl -u plc_watcher"
+            fi
         fi
 
         # ── 9. reader script path ─────────────────────────────────────────────
@@ -1258,7 +1382,8 @@ update)
     # ── Update systemd unit files if they changed ─────────────────────────────
     _UNITS_UPDATED=0
     for _svc_src in "${INSTALL_DIR}/plc_watcher.service" \
-                    "${INSTALL_DIR}/web/plc_web.service"; do
+                    "${INSTALL_DIR}/web/plc_web.service" \
+                    "${INSTALL_DIR}/plc_selfheal.service"; do
         [[ ! -f "$_svc_src" ]] && continue
         _svc_dst="/etc/systemd/system/$(basename "$_svc_src")"
         if [[ ! -f "$_svc_dst" ]] || ! diff -q "$_svc_src" "$_svc_dst" &>/dev/null; then
@@ -1271,6 +1396,22 @@ update)
         spin_ok "systemd reloaded"
     fi
 
+    # ── Self-healing daemon present + enabled (retrofit older installs) ────────
+    spin_start "Self-healing daemon"
+    if [[ -f "${INSTALL_DIR}/selfheal.py" && -f /etc/systemd/system/plc_selfheal.service ]]; then
+        if ! systemctl is-enabled --quiet plc_selfheal 2>/dev/null; then
+            sudo systemctl enable plc_selfheal 2>/dev/null || true
+            spin_ok "plc_selfheal enabled"
+        elif ! systemctl is-active --quiet plc_selfheal 2>/dev/null; then
+            sudo systemctl start plc_selfheal 2>/dev/null || true
+            spin_ok "plc_selfheal started"
+        else
+            spin_ok "plc_selfheal active"
+        fi
+    else
+        spin_warn "selfheal.py or unit missing — will be installed on next full setup"
+    fi
+
     # ── Update Plymouth boot splash ───────────────────────────────────────────
     _THEME_DIR="/usr/share/plymouth/themes/saismruth"
     _LOGO_SRC="${INSTALL_DIR}/assets/logo.png"
@@ -1528,6 +1669,7 @@ SUDOEOF
     if [[ $_CODE_CHANGED -eq 1 || $_UNITS_UPDATED -eq 1 ]]; then
         spin_start "Restarting services"
         sudo systemctl restart plc_watcher plc_web 2>/dev/null
+        sudo systemctl restart plc_selfheal 2>/dev/null || true
         sleep 2
         spin_ok "Services restarted"
     else
@@ -1661,12 +1803,13 @@ uninstall)
     # ── 1. Stop and disable services ─────────────────────────────────────────
     echo ""
     ustep "Stopping and disabling services"
-    for SVC in plc_watcher plc_web; do
+    for SVC in plc_selfheal plc_watcher plc_web; do
         systemctl is-active  --quiet "$SVC" 2>/dev/null && sudo systemctl stop    "$SVC" 2>/dev/null || true
         systemctl is-enabled --quiet "$SVC" 2>/dev/null && sudo systemctl disable "$SVC" 2>/dev/null || true
     done
     sudo rm -f /etc/systemd/system/plc_watcher.service \
-               /etc/systemd/system/plc_web.service
+               /etc/systemd/system/plc_web.service \
+               /etc/systemd/system/plc_selfheal.service
     spin_ok "Services removed"
 
     # ── 2. System drop-ins ───────────────────────────────────────────────────
@@ -1820,11 +1963,14 @@ help|--help|-h)
     echo "    fix -errors                Scan journal, verify RT/permissions/PLC/config"
     echo "    logs                       Stream live logs (plc_watcher + plc_web)"
     echo "    queue                      Show SMB pending queue and delivery ledger"
+    echo "    selfheal status            Self-healing daemon state + unresolved problems"
+    echo "    selfheal logs              Follow the self-heal journal"
+    echo "    selfheal now               Run one self-heal sweep immediately"
     echo ""
     echo -e "  ${W}Services${NC}"
-    echo "    start                      Start plc_watcher and plc_web"
-    echo "    stop                       Stop both services"
-    echo "    restart                    Restart both services"
+    echo "    start                      Start watcher, web and self-heal"
+    echo "    stop                       Stop all services"
+    echo "    restart                    Restart all services"
     echo ""
     echo -e "  ${W}Network${NC}"
     echo "    wifi                       Scan and switch WiFi network"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "plc-checkweigher",
-  "version": "1.32.2",
+  "version": "1.33.0",
   "description": "One-command installer for the PLC Check-Weigher system on Raspberry Pi (PREEMPT_RT kernel, Python stack, WiFi, SMB, systemd RT services)",
   "scripts": {
     "postinstall": "node bin/cli.js"

package/setup.sh

@@ -432,9 +432,37 @@ WantedBy=multi-user.target
 EOF
     ok "plc_web.service       (Nice=-10)"
 
+    # ── Self-healing daemon — auto-repairs services/files/configs at runtime ──
+    cat > /etc/systemd/system/plc_selfheal.service << EOF
+[Unit]
+Description=PLC Check-Weigher Self-Healing Daemon
+After=network.target plc_watcher.service plc_web.service
+Wants=plc_watcher.service plc_web.service
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=${INSTALL_DIR}
+Environment=PYTHONUNBUFFERED=1
+ExecStart=${VENV_DIR}/bin/python3 -u ${INSTALL_DIR}/selfheal.py
+Restart=always
+RestartSec=10
+Nice=10
+CPUAffinity=0 1 2
+IOSchedulingClass=idle
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target
+EOF
+    cp /etc/systemd/system/plc_selfheal.service "${INSTALL_DIR}/plc_selfheal.service"
+    chown "${PI_USER}:${PI_USER}" "${INSTALL_DIR}/plc_selfheal.service"
+    ok "plc_selfheal.service  (auto-repair · cores 0-2 · Nice=10)"
+
     systemctl daemon-reload
-    systemctl enable plc_watcher.service plc_web.service
-    ok "Both services enabled — start automatically after reboot"
+    systemctl enable plc_watcher.service plc_web.service plc_selfheal.service
+    ok "All services enabled — start automatically after reboot"
 
     cp /etc/systemd/system/plc_watcher.service "${INSTALL_DIR}/plc_watcher.service"
     chown "${PI_USER}:${PI_USER}" "${INSTALL_DIR}/plc_watcher.service"