From 12b43a5f9978351c5f2bc1e720743cc8dd666de1 Mon Sep 17 00:00:00 2001
From: Alexander Zielonka <zielonka@zeasy.software>
Date: Sun, 21 Jun 2026 20:07:23 +0200
Subject: [PATCH] Fix flapping online/offline status for powered-off hosts

Cached SSH connections had no keepalive, so when a VM/host is powered off the socket stays 'connected' (~2h OS TCP timeout) and execCommand hangs with no timeout, stalling status aggregation and leaving the dashboard on a stale 'online'.

Add keepaliveInterval/keepaliveCountMax to ssh.connect (dead peers detected in ~15s). Wrap getServerStatus/getProcessUptime in an 8s timeout that drops the dead cached connection and trips the failed-host breaker, so the fast 'unreachable' path kicks in. Affects all servers (V Rising, Palworld, etc.).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 gsm-backend/services/ssh.js | 42 ++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/gsm-backend/services/ssh.js b/gsm-backend/services/ssh.js
index c0f3e75..3d0ec82 100644
--- a/gsm-backend/services/ssh.js
+++ b/gsm-backend/services/ssh.js
@@ -8,6 +8,18 @@ const sshConnections = new Map();
 const failedHosts = new Map(); // Cache failed connections
 const FAILED_HOST_TTL = 60000; // 60 seconds before retry
 const SSH_TIMEOUT = 5000;
+const STATUS_TIMEOUT = 8000; // max time for a status/uptime check before treating the host as unreachable
+
+// Race a promise against a timeout. Rejects with `label` if it doesn't settle in time.
+// Used so a hung SSH command (e.g. host powered off mid-session, socket not yet dead)
+// can't stall the whole status aggregation.
+function withTimeout(promise, ms, label = "operation-timed-out") {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(label)), ms);
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
 
 function loadConfig() {
   return JSON.parse(readFileSync(join(__dirname, "..", "config.json"), "utf-8"));
@@ -52,7 +64,12 @@ async function getConnection(host, username = "root") {
       host,
       username,
       privateKeyPath: "/root/.ssh/id_ed25519",
-      readyTimeout: SSH_TIMEOUT
+      readyTimeout: SSH_TIMEOUT,
+      // Detect dead peers fast: if a host is powered off, the TCP socket would
+      // otherwise stay "connected" for ~2h. Keepalive closes it in ~15s so the
+      // cached connection is invalidated and the next check fails fast -> offline.
+      keepaliveInterval: 5000,
+      keepaliveCountMax: 3
     });
     clearHostFailed(host, username);
     sshConnections.set(key, ssh);
@@ -65,7 +82,8 @@ async function getConnection(host, username = "root") {
 
 // Returns: "online", "starting", "stopping", "offline"
 export async function getServerStatus(server) {
-  try {
+  const connKey = (server.sshUser || "root") + "@" + server.host;
+  const check = (async () => {
     const ssh = await getConnection(server.host, server.sshUser);
 
     if (server.runtime === 'docker') {
@@ -113,8 +131,17 @@ export async function getServerStatus(server) {
       }
       return 'offline';
     }
+  })();
+  check.catch(() => {}); // prevent unhandled rejection if the timeout wins the race
+  try {
+    return await withTimeout(check, STATUS_TIMEOUT, "status-timeout");
   } catch (err) {
     console.error(`Failed to get status for ${server.id}:`, err.message);
+    if (err.message === "status-timeout") {
+      // Connection hung (host likely powered off mid-session) -> drop it and trip the breaker
+      sshConnections.delete(connKey);
+      markHostFailed(server.host, server.sshUser);
+    }
     return 'offline';
   }
 }
@@ -257,7 +284,8 @@ export async function getConsoleLog(server, lines = 50) {
 }
 
 export async function getProcessUptime(server) {
-  try {
+  const connKey = (server.sshUser || "root") + "@" + server.host;
+  const check = (async () => {
     const ssh = await getConnection(server.host, server.sshUser);
 
     if (server.runtime === "docker") {
@@ -296,8 +324,16 @@ export async function getProcessUptime(server) {
       if (!isNaN(uptime)) return uptime;
     }
     return 0;
+  })();
+  check.catch(() => {}); // prevent unhandled rejection if the timeout wins the race
+  try {
+    return await withTimeout(check, STATUS_TIMEOUT, "uptime-timeout");
   } catch (err) {
     console.error(`Failed to get uptime for ${server.id}:`, err.message);
+    if (err.message === "uptime-timeout") {
+      sshConnections.delete(connKey);
+      markHostFailed(server.host, server.sshUser);
+    }
     return 0;
   }
 }