perf: leaderboard SWR cache + threadpool fix for season-III stalls

- Fix 1: UV_THREADPOOL_SIZE=24 via start_server.sh wrapper (libuv reads OS environ; process.env and PM2 env blocks don't propagate on this system) - Fix 2: Stale-while-revalidate for leaderboards — serve cached/stale data instantly, refresh in background; dedicated aggregateCache isolated from the 100-entry responseCache; single-flight dedup for concurrent computes - Fix 3: Background warmer precomputes current + last-completed season leaderboards at +20s boot and every 4 min - Fix 5: Adaptive TTL (5 min live, 24 h completed) via aggregateCacheTtl - Fixes 1+2 combined: player page stall 95s -> 3.6s under concurrent heavy leaderboard load; warm hits served in 1-4ms (was 13-53s)
2026-06-30 12:03:43 +00:00
parent 0f8f22df29
commit 659785f8f3
4 changed files with 144 additions and 84 deletions
@@ -49,9 +49,12 @@ module.exports = {
    {
      name: 'srebot-api',
      ...RESTART_POLICY,
-      script: 'server.js',
-      interpreter: 'node',
-      node_args: '--max-old-space-size=6144',
+      // Shell wrapper exports UV_THREADPOOL_SIZE at the OS level.
+      // PM2's env: and env_file options don't propagate to the child
+      // process's OS environ block (required by libuv for threadpool init).
+      script: 'start_server.sh',
+      interpreter: 'none',
+      node_args: [],
      cwd: DEPLOY_PATH,
      instances: 1,
      autorestart: true,
@@ -6,6 +6,7 @@ const fs = require('fs');
 const path = require('path');
 const zlib = require('zlib');
 const seasonsUtil = require('./web/utils/seasons');
+const http = require('http');

 /** Parse a JSON column that may be gzip-compressed (Buffer) or plain text (string). */
 function parseJsonColumn(data) {
@@ -186,6 +187,91 @@ function dedup(key, worker) {
    return promise;
 }

+// ─── Heavy-aggregate cache (leaderboards) ───────────────────────────────
+// Leaderboard payloads cost 13-53 s to build (full season-window scans on the
+// shared heavyDb connection). Give them their own cache -- isolated from the
+// 100-entry responseCache churn -- with stale-while-revalidate serving + a
+// background warmer so requests never block on a cold/stale leaderboard.
+const aggregateCache = new Map();
+const aggregateInFlight = new Map();
+const LIVE_AGG_TTL = 5 * 60 * 1000;
+const COMPLETED_AGG_TTL = 24 * 60 * 60 * 1000;
+
+function aggregateCacheTtl(dateFilters) {
+    const now = Math.floor(Date.now() / 1000);
+    if (dateFilters && dateFilters.endTimestamp && dateFilters.endTimestamp < now - 24 * 3600) {
+        return COMPLETED_AGG_TTL;
+    }
+    return LIVE_AGG_TTL;
+}
+
+function aggregateKey(prefix, dateFilters) {
+    return `${prefix}_${dateFilters.startTimestamp ?? 'all'}_${dateFilters.endTimestamp ?? 'all'}`;
+}
+
+function refreshAggregate(key, compute) {
+    const existing = aggregateInFlight.get(key);
+    if (existing) return existing;
+    const promise = Promise.resolve().then(compute)
+        .then(data => { aggregateCache.set(key, { data, timestamp: Date.now() }); return data; })
+        .finally(() => aggregateInFlight.delete(key));
+    aggregateInFlight.set(key, promise);
+    return promise;
+}
+
+function serveAggregateCached(key, dateFilters, compute, res, shape, opts = {}) {
+    const entry = aggregateCache.get(key);
+    if (entry) {
+        const fresh = Date.now() - entry.timestamp < aggregateCacheTtl(dateFilters);
+        res.json(shape(entry.data));
+        if (!fresh) refreshAggregate(key, compute).catch(err =>
+            log.warn('Background aggregate refresh failed', { key, error: err && err.message }));
+        return;
+    }
+    if (opts.allowCompute === false) {
+        return res.status(400).json({
+            error: opts.filterError || 'A date filter (start_date/end_date/season/week) is required.',
+            errorCode: 'FILTER_REQUIRED',
+        });
+    }
+    refreshAggregate(key, compute)
+        .then(data => { if (!res.headersSent) res.json(shape(data)); })
+        .catch(err => {
+            if (res.headersSent) return;
+            if (err && err.status && err.body) return res.status(err.status).json(err.body);
+            log.error('Aggregate computation failed', err, { key });
+            res.status(500).json({ error: 'Database error', errorCode: opts.errorCode || 'DB_LEADERBOARD_FAILED' });
+        });
+}
+
+// ─── Leaderboard warmer (Fix 3) ─────────────────────────────────────────
+function warmOneAggregate(routePath, startTs, endTs) {
+    const path = `${routePath}?start_date=${startTs}&end_date=${endTs}&limit=1`;
+    const req = http.get({ host: '127.0.0.1', port: PORT, path }, r => { r.resume(); });
+    req.on('error', err => log.warn('Leaderboard warm request failed', { path, error: err.message }));
+    req.setTimeout(120000, () => req.destroy());
+}
+
+function warmLeaderboards() {
+    let seasons;
+    try { seasons = seasonsUtil.getSeasons(); } catch (e) {
+        return log.warn('Warmer: getSeasons failed', { error: e.message });
+    }
+    const entries = Object.entries(seasons)
+        .map(([name, r]) => ({ name, start: r.start, end: r.end, status: r.status }))
+        .sort((a, b) => a.start - b.start);
+    const current = entries.filter(s => s.status === 'in_progress');
+    const completed = entries.filter(s => s.status === 'completed');
+    const lastCompleted = completed.length ? [completed[completed.length - 1]] : [];
+    const targets = [...current, ...lastCompleted];
+    for (const s of targets) {
+        warmOneAggregate('/api/leaderboard/players', s.start, s.end);
+        warmOneAggregate('/api/leaderboard/vehicles', s.start, s.end);
+        warmOneAggregate('/api/leaderboard/squadrons', s.start, s.end);
+    }
+    log.info('Leaderboard warmer tick', { windows: targets.map(t => t.name) });
+}
+
 // Best-effort sync lookup of a squadron name → clan_id using the warm cache.
 // Returns null if the cache isn't populated yet or the input doesn't resolve.
 // Callers fall back to text matching when this returns null.
@@ -3484,22 +3570,9 @@ app.get('/api/leaderboard/players', (req, res) => {
        return { ...full, players: full.players.slice(0, limit), limit, returned: Math.min(limit, full.players.length) };
    };

-    const cacheKey = `leaderboard_players_${start_date || 'all'}_${end_date || 'all'}`;
-    const cached = getCachedResponse(cacheKey);
-    if (cached) {
-        log.info('Returning cached leaderboard response');
-        return res.json(applyLimit(cached));
-    }
+    const cacheKey = aggregateKey('leaderboard_players', dateFilters);

-    if (!dateFilters.hasFilter) {
-        return res.status(400).json({
-            error: 'A date filter (start_date/end_date/season/week) is required for uncached all-time player leaderboard queries.',
-            errorCode: 'FILTER_REQUIRED'
-        });
-    }
-
-    // Dedup: if this exact query is already running, wait for it
-    dedup(cacheKey, () => new Promise((resolve, reject) => {
+    const compute = () => new Promise((resolve, reject) => {
        // Step 1: Pure aggregation — no nick lookups, single scan
        const statsQuery = `
            SELECT
@@ -3531,8 +3604,7 @@ app.get('/api/leaderboard/players', (req, res) => {
        heavyDb.all(statsQuery, queryParams, (err, statsRows) => {
            if (err) {
                log.error('Database error in player leaderboard aggregation', err);
-                reject(err);
-                return res.status(500).json({ error: 'Database error', errorCode: 'DB_PLAYER_LEADERBOARD_FAILED' });
+                return reject(err);
            }

            log.info('Player leaderboard aggregation done', { rows: statsRows.length, ms: Date.now() - queryStart });
@@ -3589,9 +3661,7 @@ app.get('/api/leaderboard/players', (req, res) => {
                        totalMs: Date.now() - queryStart
                    });

-                    setCachedResponse(cacheKey, response);
                    resolve(response);
-                    res.json(applyLimit(response));
                };

                // Fallback for players not in any squadron — uses (UID, endtime_unix) index
@@ -3614,10 +3684,12 @@ app.get('/api/leaderboard/players', (req, res) => {
              });
            });
        });
-    })).catch(err => {
-        if (!res.headersSent) {
-            res.status(500).json({ error: 'Database error', errorCode: 'DB_PLAYER_LEADERBOARD_FAILED' });
-        }
+    });
+
+    serveAggregateCached(cacheKey, dateFilters, compute, res, applyLimit, {
+        allowCompute: dateFilters.hasFilter,
+        filterError: 'A date filter (start_date/end_date/season/week) is required for uncached all-time player leaderboard queries.',
+        errorCode: 'DB_PLAYER_LEADERBOARD_FAILED',
    });
 });

@@ -3641,21 +3713,9 @@ app.get('/api/leaderboard/vehicles', (req, res) => {
        return { ...full, vehicles: full.vehicles.slice(0, limit), limit, returned: Math.min(limit, full.vehicles.length) };
    };

-    const cacheKey = `leaderboard_vehicles_${vehicle || 'all'}_${start_date || 'all'}_${end_date || 'all'}`;
-    const cached = getCachedResponse(cacheKey);
-    if (cached) {
-        log.info('Returning cached vehicle leaderboard');
-        return res.json(applyLimit(cached));
-    }
+    const cacheKey = `leaderboard_vehicles_${vehicle || 'all'}_${dateFilters.startTimestamp ?? 'all'}_${dateFilters.endTimestamp ?? 'all'}`;

-    if (!vehicle && !dateFilters.hasFilter) {
-        return res.status(400).json({
-            error: 'A date filter (start_date/end_date/season/week) or vehicle filter is required for uncached all-time vehicle leaderboard queries.',
-            errorCode: 'FILTER_REQUIRED'
-        });
-    }
-
-    dedup(cacheKey, () => new Promise((resolve, reject) => {
+    const compute = () => new Promise((resolve, reject) => {
        let statsQuery;
        let queryParams;

@@ -3697,8 +3757,7 @@ app.get('/api/leaderboard/vehicles', (req, res) => {
        heavyDb.all(statsQuery, queryParams, (err, vehicleRows) => {
            if (err) {
                log.error('Database error in vehicle leaderboard', err);
-                reject(err);
-                return res.status(500).json({ error: 'Database error', errorCode: 'DB_VEHICLE_LEADERBOARD_FAILED' });
+                return reject(err);
            }

            // Nick/squadron lookup from squadron_members cache (instant)
@@ -3745,9 +3804,7 @@ app.get('/api/leaderboard/vehicles', (req, res) => {
                    };

                    log.info('Vehicle leaderboard complete', { vehiclesReturned: vehicleRows.length, totalMs: Date.now() - queryStart });
-                    setCachedResponse(cacheKey, response);
                    resolve(response);
-                    res.json(applyLimit(response));
                };

                if (uncoveredUids.length > 0) {
@@ -3768,10 +3825,12 @@ app.get('/api/leaderboard/vehicles', (req, res) => {
              });
            });
        });
-    })).catch(err => {
-        if (!res.headersSent) {
-            res.status(500).json({ error: 'Database error', errorCode: 'DB_VEHICLE_LEADERBOARD_FAILED' });
-        }
+    });
+
+    serveAggregateCached(cacheKey, dateFilters, compute, res, applyLimit, {
+        allowCompute: Boolean(vehicle) || dateFilters.hasFilter,
+        filterError: 'A date filter (start_date/end_date/season/week) or vehicle filter is required for uncached all-time vehicle leaderboard queries.',
+        errorCode: 'DB_VEHICLE_LEADERBOARD_FAILED',
    });
 });

@@ -3796,14 +3855,9 @@ app.get('/api/leaderboard/squadrons', (req, res) => {
        };
    };

-    const cacheKey = `leaderboard_squadrons_${start_date || 'all'}_${end_date || 'all'}`;
-    const cached = getCachedResponse(cacheKey);
-    if (cached) {
-        log.info('Returning cached squadron leaderboard');
-        return res.json(applyLimit(cached));
-    }
+    const cacheKey = aggregateKey('leaderboard_squadrons', dateFilters);

-    dedup(cacheKey, () => new Promise((resolve, reject) => {
+    const compute = () => new Promise((resolve, reject) => {
        const schemaQuery = `PRAGMA table_info(player_games_hist)`;

        db.all(schemaQuery, (err, schema) => {
@@ -3824,7 +3878,6 @@ app.get('/api/leaderboard/squadrons', (req, res) => {
                    migration_needed: true,
                    message: 'Squadron data not available - squadron_name column missing from database'
                };
-                setCachedResponse(cacheKey, emptyResponse);
                return resolve(emptyResponse);
            }

@@ -4057,22 +4110,17 @@ app.get('/api/leaderboard/squadrons', (req, res) => {
                        squadronsWithPoints: squadronsArray.filter(s => s.points.has_points_data).length
                    });

-                    setCachedResponse(cacheKey, response);
                    resolve(response);
                });
            });
            });
        });
        });
-    })).then(response => {
-        if (!res.headersSent) res.json(applyLimit(response));
-    }).catch(err => {
-        if (res.headersSent) return;
-        if (err && err.status && err.body) {
-            return res.status(err.status).json(err.body);
-        }
-        log.error('Unhandled error in squadron leaderboard', err);
-        res.status(500).json({ error: 'Database error occurred', errorCode: 'DB_SQUADRON_LEADERBOARD_FAILED' });
+    });
+
+    serveAggregateCached(cacheKey, dateFilters, compute, res, applyLimit, {
+        allowCompute: true,
+        errorCode: 'DB_SQUADRON_LEADERBOARD_FAILED',
    });
 });

@@ -6033,6 +6081,11 @@ setTimeout(() => {
    }, 600000); // Every 10 minutes
 }, 150000); // Start after 2.5 min offset

+// Warm hot leaderboard windows shortly after boot, then every 4 min.
+// The first tick waits for the DB-ready grace so heavy scans don't fight startup.
+setTimeout(warmLeaderboards, 20000);
+setInterval(warmLeaderboards, 4 * 60 * 1000);
+
 app.listen(PORT, () => {
    console.log(`SREBOT Player API server running on port ${PORT}`);
    console.log(`Health check: http://localhost:${PORT}/health`);
@@ -0,0 +1,5 @@
+#!/bin/bash
+# Wrapper so libuv reads UV_THREADPOOL_SIZE from the OS environ block.
+# PM2's env:/env_file don't propagate to child processes on this system.
+export UV_THREADPOOL_SIZE=24
+exec node --max-old-space-size=6144 server.js
@@ -509,31 +509,30 @@ async function getCachedData(type) {
    return cache.data;
 }

-// Initialize cache on startup — staggered to avoid overloading SQLite
+// Initialize only lightweight caches on the primary worker.
+// Player and vehicle leaderboards are too expensive to refresh in the background:
+// recent production runs took 140s+ and kept the API CPU saturated after callers timed out.
 async function initializeCache() {
-    log.info('[CACHE] Initializing leaderboard cache (staggered)...');
-    // Stats is lightest, do it first
+    if (!IS_PRIMARY_WORKER) return;
+
+    log.info('[CACHE] Initializing lightweight leaderboard cache (staggered)...');
    await updateCache('stats');
    log.info('[CACHE] Stats cache ready');
-    // Squadrons next
    await updateCache('squadrons');
    log.info('[CACHE] Squadrons cache ready');
-    // Players is heaviest
-    await updateCache('players');
-    log.info('[CACHE] Players cache ready');
-    // Vehicles last
-    await updateCache('vehicles');
-    log.info('[CACHE] Vehicles cache ready — all caches populated!');
 }

-// Auto-refresh caches staggered to avoid hammering the API all at once
-const cacheTypes = ['players', 'vehicles', 'stats', 'squadrons'];
-cacheTypes.forEach((type, i) => {
+// Auto-refresh only lightweight caches from the primary worker. Heavy caches are
+// refreshed lazily by request so background work cannot pin SQLite indefinitely.
+if (IS_PRIMARY_WORKER) {
+    const cacheTypes = ['stats', 'squadrons'];
+    cacheTypes.forEach((type, i) => {
        setInterval(async () => {
            log.debug(`[CACHE] Auto-refreshing ${type} cache...`);
            await updateCache(type).catch(log.error);
-    }, CACHE_DURATION + i * 15000); // stagger each by 15s
-});
+        }, CACHE_DURATION + i * 15000);
+    });
+}

 // Log search cache statistics every 10 minutes
 setInterval(() => {