From 024d97e5442b33ac5ed0d6be8165405fc32ca393 Mon Sep 17 00:00:00 2001 From: Alexander Zielonka Date: Fri, 26 Jun 2026 13:46:34 +0200 Subject: [PATCH] Speed up server list & detail loads Three compounding causes made the dashboard and detail pages slow: - Detail page fetched the WHOLE fleet (every SSH/Prometheus/RCON call for all servers) just to show one. Switch ServerDetail to GET /servers/:id (add api.getServer) so it costs one server's worth of work, not N. - No server-side caching: every 10s poll, from every open tab, recomputed everything. Add a shared 5s-TTL per-server cache + in-flight dedup behind both GET / and GET /:id; invalidate on start/stop/restart so power actions still reflect immediately. - Prometheus fetch had no timeout, so a slow/restarting Prometheus hung the whole aggregation indefinitely (the .catch can't catch a hang). Add AbortSignal.timeout to queryPrometheus/queryPrometheusRange. Co-Authored-By: Claude Opus 4.8 (1M context) --- gsm-backend/routes/servers.js | 255 +++++++++++------------- gsm-backend/services/prometheus.js | 8 +- gsm-frontend/src/api.js | 8 + gsm-frontend/src/pages/ServerDetail.jsx | 11 +- 4 files changed, 134 insertions(+), 148 deletions(-) diff --git a/gsm-backend/routes/servers.js b/gsm-backend/routes/servers.js index 09437f7..fe3ccc5 100644 --- a/gsm-backend/routes/servers.js +++ b/gsm-backend/routes/servers.js @@ -41,6 +41,114 @@ function formatBytes(bytes, forceUnit = null) { return { value: mb, unit: "MB" }; } +// ---- Server status aggregation cache ---- +// Building a server's status payload means SSH round-trips + 6 Prometheus queries. +// Both GET / (dashboard) and GET /:id (detail page) need exactly the same data, and +// the frontend polls every 10s — often from several tabs/users at once. A short-TTL +// cache collapses all of that into one computation per server every few seconds, and +// the in-flight map prevents a stampede when several requests miss the cache together. +const SERVER_STATUS_TTL = 5000; // ms +const serverPayloadCache = new Map(); // id -> { time, data } +const serverPayloadInflight = new Map(); // id -> Promise + +function invalidateServerPayload(id) { + serverPayloadCache.delete(id); + serverPayloadInflight.delete(id); +} + +async function computeServerPayload(server) { + // Host marked unreachable by the SSH breaker -> skip the expensive SSH calls, + // still surface whatever Prometheus has (it scrapes independently of SSH). + if (isHostFailed(server.host, server.sshUser)) { + const metrics = await getCurrentMetrics(server.id).catch(() => ({ + cpu: 0, cpuCores: 1, memory: 0, memoryUsed: 0, memoryTotal: 0, uptime: 0 + })); + const memTotal = formatBytes(metrics.memoryTotal); + const memUsed = formatBytes(metrics.memoryUsed, memTotal.unit); + return { + id: server.id, + name: server.name, + type: server.type, + status: "unreachable", + running: false, + metrics: { + cpu: metrics.cpu, + cpuCores: metrics.cpuCores, + memory: metrics.memory, + memoryUsed: memUsed.value, + memoryTotal: memTotal.value, + memoryUnit: memTotal.unit, + uptime: 0 + }, + players: { online: 0, max: null, list: [] }, + hasRcon: !!server.rconPassword + }; + } + + const wantsPlayers = server.rconPassword || server.type === 'hytale' || server.type === 'terraria' || server.type === 'spaceengineers'; + const [status, metrics, players, playerList, processUptime] = await Promise.all([ + getServerStatus(server), + getCurrentMetrics(server.id).catch(() => ({ + cpu: 0, cpuCores: 1, memory: 0, memoryUsed: 0, memoryTotal: 0, uptime: 0 + })), + wantsPlayers ? getPlayers(server).catch(() => ({ online: 0, max: null })) : { online: 0, max: null }, + wantsPlayers ? getPlayerList(server).catch(() => ({ players: [] })) : { players: [] }, + getProcessUptime(server).catch(() => 0) + ]); + + const memTotal = formatBytes(metrics.memoryTotal); + const memUsed = formatBytes(metrics.memoryUsed, memTotal.unit); + const shutdownSettings = getAutoShutdownSettings(server.id); + const emptySince = getEmptySince(server.id); + + return { + id: server.id, + name: server.name, + type: server.type, + status, + running: status === 'online', + metrics: { + cpu: metrics.cpu, + cpuCores: metrics.cpuCores, + memory: metrics.memory, + memoryUsed: memUsed.value, + memoryTotal: memTotal.value, + memoryUnit: memTotal.unit, + uptime: processUptime + }, + players: { + ...players, + list: playerList.players + }, + hasRcon: !!server.rconPassword, + autoShutdown: { + enabled: shutdownSettings?.enabled === 1 || false, + timeoutMinutes: shutdownSettings?.timeout_minutes || 15, + emptySinceMinutes: emptySince + } + }; +} + +async function getServerPayload(server) { + const cached = serverPayloadCache.get(server.id); + if (cached && Date.now() - cached.time < SERVER_STATUS_TTL) { + return cached.data; + } + const inflight = serverPayloadInflight.get(server.id); + if (inflight) return inflight; + + const p = computeServerPayload(server) + .then((data) => { + serverPayloadCache.set(server.id, { time: Date.now(), data }); + return data; + }) + .finally(() => { + serverPayloadInflight.delete(server.id); + }); + serverPayloadInflight.set(server.id, p); + return p; +} + // ============ FACTORIO ROUTES ============ // Factorio: List saves @@ -315,81 +423,7 @@ router.put("/palworld/config/:filename", authenticateToken, requireRole("moderat router.get('/', optionalAuth, async (req, res) => { try { const config = loadConfig(); - const servers = await Promise.all(config.servers.map(async (server) => { - // Quick check if host is unreachable - skip expensive operations - const hostUnreachable = isHostFailed(server.host, server.sshUser); - // If host is unreachable, return immediately with minimal data - if (hostUnreachable) { - const metrics = await getCurrentMetrics(server.id).catch(() => ({ - cpu: 0, cpuCores: 1, memory: 0, memoryUsed: 0, memoryTotal: 0, uptime: 0 - })); - const memTotal = formatBytes(metrics.memoryTotal); - const memUsed = formatBytes(metrics.memoryUsed, memTotal.unit); - return { - id: server.id, - name: server.name, - type: server.type, - status: "unreachable", - running: false, - metrics: { - cpu: metrics.cpu, - cpuCores: metrics.cpuCores, - memory: metrics.memory, - memoryUsed: memUsed.value, - memoryTotal: memTotal.value, - memoryUnit: memTotal.unit, - uptime: 0 - }, - players: { online: 0, max: null, list: [] }, - hasRcon: !!server.rconPassword - }; - } - - const [status, metrics, players, playerList, processUptime] = await Promise.all([ - getServerStatus(server), - getCurrentMetrics(server.id).catch(() => ({ - cpu: 0, cpuCores: 1, memory: 0, memoryUsed: 0, memoryTotal: 0, uptime: 0 - })), - (server.rconPassword || server.type === 'hytale' || server.type === 'terraria' || server.type === 'spaceengineers') ? getPlayers(server).catch(() => ({ online: 0, max: null })) : { online: 0, max: null }, - (server.rconPassword || server.type === 'hytale' || server.type === 'terraria' || server.type === 'spaceengineers') ? getPlayerList(server).catch(() => ({ players: [] })) : { players: [] }, - getProcessUptime(server).catch(() => 0) - ]); - - const memTotal = formatBytes(metrics.memoryTotal); - const memUsed = formatBytes(metrics.memoryUsed, memTotal.unit); - - // Get auto-shutdown info - const shutdownSettings = getAutoShutdownSettings(server.id); - const emptySince = getEmptySince(server.id); - - return { - id: server.id, - name: server.name, - type: server.type, - status, - running: status === 'online', - metrics: { - cpu: metrics.cpu, - cpuCores: metrics.cpuCores, - memory: metrics.memory, - memoryUsed: memUsed.value, - memoryTotal: memTotal.value, - memoryUnit: memTotal.unit, - uptime: processUptime - }, - players: { - ...players, - list: playerList.players - }, - hasRcon: !!server.rconPassword, - autoShutdown: { - enabled: shutdownSettings?.enabled === 1 || false, - timeoutMinutes: shutdownSettings?.timeout_minutes || 15, - emptySinceMinutes: emptySince - } - }; - })); - + const servers = await Promise.all(config.servers.map((server) => getServerPayload(server))); res.json(servers); } catch (err) { res.status(500).json({ error: err.message }); @@ -546,7 +580,9 @@ router.put("/hytale/config", authenticateToken, requireRole("moderator"), async }); -// Get single server (guests not allowed) +// Get single server (guests not allowed). Shares the same short-TTL cache as the +// dashboard list, so the detail page costs one server's worth of work — not the +// whole fleet — and a user with both pages open reuses the same computation. router.get('/:id', authenticateToken, rejectGuest, async (req, res) => { const config = loadConfig(); const server = config.servers.find(s => s.id === req.params.id); @@ -555,68 +591,8 @@ router.get('/:id', authenticateToken, rejectGuest, async (req, res) => { } try { - // Check if host is unreachable - const hostUnreachable = isHostFailed(server.host, server.sshUser); - if (hostUnreachable) { - const metrics = await getCurrentMetrics(server.id).catch(() => ({ - cpu: 0, cpuCores: 1, memory: 0, memoryUsed: 0, memoryTotal: 0, uptime: 0 - })); - const memTotal = formatBytes(metrics.memoryTotal); - const memUsed = formatBytes(metrics.memoryUsed, memTotal.unit); - return res.json({ - id: server.id, - name: server.name, - type: server.type, - status: "unreachable", - running: false, - metrics: { - cpu: metrics.cpu, - cpuCores: metrics.cpuCores, - memory: metrics.memory, - memoryUsed: memUsed.value, - memoryTotal: memTotal.value, - memoryUnit: memTotal.unit, - uptime: 0 - }, - players: { online: 0, max: null, list: [] }, - hasRcon: !!server.rconPassword - }); - } - - const [status, metrics, players, playerList, processUptime] = await Promise.all([ - getServerStatus(server), - getCurrentMetrics(server.id).catch(() => ({ - cpu: 0, cpuCores: 1, memory: 0, memoryUsed: 0, memoryTotal: 0, uptime: 0 - })), - (server.rconPassword || server.type === 'hytale' || server.type === 'terraria' || server.type === 'spaceengineers') ? getPlayers(server).catch(() => ({ online: 0, max: null })) : { online: 0, max: null }, - (server.rconPassword || server.type === 'hytale' || server.type === 'terraria' || server.type === 'spaceengineers') ? getPlayerList(server).catch(() => ({ players: [] })) : { players: [] }, - getProcessUptime(server).catch(() => 0) - ]); - - const memTotal = formatBytes(metrics.memoryTotal); - const memUsed = formatBytes(metrics.memoryUsed, memTotal.unit); - - res.json({ - id: server.id, - name: server.name, - type: server.type, - status, - running: status === 'online', - metrics: { - cpu: metrics.cpu, - cpuCores: metrics.cpuCores, - memory: metrics.memory, - memoryUsed: memUsed.value, - memoryTotal: memTotal.value, - memoryUnit: memTotal.unit, - uptime: processUptime - }, - players: { - ...players, - list: playerList.players - }, - hasRcon: !!server.rconPassword - }); + const payload = await getServerPayload(server); + res.json(payload); } catch (err) { console.error(`Error fetching server ${req.params.id}:`, err.message); res.status(500).json({ error: err.message }); @@ -725,6 +701,7 @@ router.post('/:id/start', authenticateToken, requireRole('moderator'), async (re try { const { save } = req.body || {}; await startServer(server, { save }); + invalidateServerPayload(server.id); // force a fresh status on the next poll logActivity(req.user.id, req.user.username, 'server_start', server.id, save ? 'Save: ' + save : null, req.user.discordId, req.user.avatar); res.json({ message: 'Server starting' }); } catch (err) { @@ -746,6 +723,7 @@ router.post('/:id/stop', authenticateToken, requireRole('moderator'), async (req try { await stopServer(server); + invalidateServerPayload(server.id); // force a fresh status on the next poll logActivity(req.user.id, req.user.username, 'server_stop', server.id, null, req.user.discordId, req.user.avatar); res.json({ message: 'Server stopping' }); } catch (err) { @@ -767,6 +745,7 @@ router.post('/:id/restart', authenticateToken, requireRole('moderator'), async ( try { await restartServer(server); + invalidateServerPayload(server.id); // force a fresh status on the next poll logActivity(req.user.id, req.user.username, 'server_restart', server.id, null, req.user.discordId, req.user.avatar); res.json({ message: 'Server restarting' }); } catch (err) { diff --git a/gsm-backend/services/prometheus.js b/gsm-backend/services/prometheus.js index 86c1914..c99b441 100644 --- a/gsm-backend/services/prometheus.js +++ b/gsm-backend/services/prometheus.js @@ -1,6 +1,10 @@ import fetch from "node-fetch"; const PROMETHEUS_URL = "http://localhost:9090"; +// Without a timeout a slow/restarting Prometheus makes fetch() hang forever; the +// surrounding .catch() can't help because a hung request never rejects. Abort +// after a few seconds so the status aggregation can't stall on Prometheus. +const PROMETHEUS_TIMEOUT_MS = 4000; const SERVER_JOBS = { "vrising": "vrising", @@ -16,7 +20,7 @@ const SERVER_JOBS = { export async function queryPrometheus(query) { const url = `${PROMETHEUS_URL}/api/v1/query?query=${encodeURIComponent(query)}`; - const res = await fetch(url); + const res = await fetch(url, { signal: AbortSignal.timeout(PROMETHEUS_TIMEOUT_MS) }); const data = await res.json(); if (data.status !== "success") { throw new Error(`Prometheus query failed: ${data.error}`); @@ -26,7 +30,7 @@ export async function queryPrometheus(query) { export async function queryPrometheusRange(query, start, end, step) { const url = `${PROMETHEUS_URL}/api/v1/query_range?query=${encodeURIComponent(query)}&start=${start}&end=${end}&step=${step}`; - const res = await fetch(url); + const res = await fetch(url, { signal: AbortSignal.timeout(PROMETHEUS_TIMEOUT_MS) }); const data = await res.json(); if (data.status !== "success") { throw new Error(`Prometheus query failed: ${data.error}`); diff --git a/gsm-frontend/src/api.js b/gsm-frontend/src/api.js index efa1e7f..3bc93b6 100644 --- a/gsm-frontend/src/api.js +++ b/gsm-frontend/src/api.js @@ -131,6 +131,14 @@ export async function getServers(token) { }) } +// Fetch a single server's status payload. Cheaper than getServers() — only one +// server is queried server-side instead of the whole fleet. +export async function getServer(token, serverId) { + return fetchAPI(`/servers/${serverId}`, { + headers: { Authorization: `Bearer ${token}` }, + }) +} + export async function serverAction(token, serverId, action, body = null) { return fetchAPI(`/servers/${serverId}/${action}`, { method: 'POST', diff --git a/gsm-frontend/src/pages/ServerDetail.jsx b/gsm-frontend/src/pages/ServerDetail.jsx index a04f5a9..f8f5a96 100644 --- a/gsm-frontend/src/pages/ServerDetail.jsx +++ b/gsm-frontend/src/pages/ServerDetail.jsx @@ -1,6 +1,6 @@ import { useState, useEffect, useRef } from 'react' import { useParams, useNavigate } from 'react-router-dom' -import { getServers, serverAction, sendRcon, getServerLogs, getWhitelist, getFactorioCurrentSave, getAutoShutdownSettings, setAutoShutdownSettings, getDisplaySettings, setDisplaySettings, getServerVersion } from '../api' +import { getServer, serverAction, sendRcon, getServerLogs, getWhitelist, getFactorioCurrentSave, getAutoShutdownSettings, setAutoShutdownSettings, getDisplaySettings, setDisplaySettings, getServerVersion } from '../api' import { useUser } from '../context/UserContext' import MetricsChart from '../components/MetricsChart' import ConfirmModal from '../components/ConfirmModal' @@ -73,13 +73,8 @@ export default function ServerDetail() { const fetchServer = async () => { try { - const servers = await getServers(token) - const found = servers.find(s => s.id === serverId) - if (found) { - setServer(found); document.title = found.name + " | Zeasy GSM" - } else { - navigate('/') - } + const found = await getServer(token, serverId) + setServer(found); document.title = found.name + " | Zeasy GSM" } catch (err) { console.error(err) navigate('/')