diff --git a/ecosystem.config.js b/ecosystem.config.js index f0d2db7..fb7f0c3 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -49,9 +49,12 @@ module.exports = { { name: 'srebot-api', ...RESTART_POLICY, - script: 'server.js', - interpreter: 'node', - node_args: '--max-old-space-size=6144', + // Shell wrapper exports UV_THREADPOOL_SIZE at the OS level. + // PM2's env: and env_file options don't propagate to the child + // process's OS environ block (required by libuv for threadpool init). + script: 'start_server.sh', + interpreter: 'none', + node_args: [], cwd: DEPLOY_PATH, instances: 1, autorestart: true, diff --git a/server.js b/server.js index 79c9033..e12dcc5 100644 --- a/server.js +++ b/server.js @@ -6,6 +6,7 @@ const fs = require('fs'); const path = require('path'); const zlib = require('zlib'); const seasonsUtil = require('./web/utils/seasons'); +const http = require('http'); /** Parse a JSON column that may be gzip-compressed (Buffer) or plain text (string). */ function parseJsonColumn(data) { @@ -186,6 +187,91 @@ function dedup(key, worker) { return promise; } +// ─── Heavy-aggregate cache (leaderboards) ─────────────────────────────── +// Leaderboard payloads cost 13-53 s to build (full season-window scans on the +// shared heavyDb connection). Give them their own cache -- isolated from the +// 100-entry responseCache churn -- with stale-while-revalidate serving + a +// background warmer so requests never block on a cold/stale leaderboard. +const aggregateCache = new Map(); +const aggregateInFlight = new Map(); +const LIVE_AGG_TTL = 5 * 60 * 1000; +const COMPLETED_AGG_TTL = 24 * 60 * 60 * 1000; + +function aggregateCacheTtl(dateFilters) { + const now = Math.floor(Date.now() / 1000); + if (dateFilters && dateFilters.endTimestamp && dateFilters.endTimestamp < now - 24 * 3600) { + return COMPLETED_AGG_TTL; + } + return LIVE_AGG_TTL; +} + +function aggregateKey(prefix, dateFilters) { + return `${prefix}_${dateFilters.startTimestamp ?? 'all'}_${dateFilters.endTimestamp ?? 'all'}`; +} + +function refreshAggregate(key, compute) { + const existing = aggregateInFlight.get(key); + if (existing) return existing; + const promise = Promise.resolve().then(compute) + .then(data => { aggregateCache.set(key, { data, timestamp: Date.now() }); return data; }) + .finally(() => aggregateInFlight.delete(key)); + aggregateInFlight.set(key, promise); + return promise; +} + +function serveAggregateCached(key, dateFilters, compute, res, shape, opts = {}) { + const entry = aggregateCache.get(key); + if (entry) { + const fresh = Date.now() - entry.timestamp < aggregateCacheTtl(dateFilters); + res.json(shape(entry.data)); + if (!fresh) refreshAggregate(key, compute).catch(err => + log.warn('Background aggregate refresh failed', { key, error: err && err.message })); + return; + } + if (opts.allowCompute === false) { + return res.status(400).json({ + error: opts.filterError || 'A date filter (start_date/end_date/season/week) is required.', + errorCode: 'FILTER_REQUIRED', + }); + } + refreshAggregate(key, compute) + .then(data => { if (!res.headersSent) res.json(shape(data)); }) + .catch(err => { + if (res.headersSent) return; + if (err && err.status && err.body) return res.status(err.status).json(err.body); + log.error('Aggregate computation failed', err, { key }); + res.status(500).json({ error: 'Database error', errorCode: opts.errorCode || 'DB_LEADERBOARD_FAILED' }); + }); +} + +// ─── Leaderboard warmer (Fix 3) ───────────────────────────────────────── +function warmOneAggregate(routePath, startTs, endTs) { + const path = `${routePath}?start_date=${startTs}&end_date=${endTs}&limit=1`; + const req = http.get({ host: '127.0.0.1', port: PORT, path }, r => { r.resume(); }); + req.on('error', err => log.warn('Leaderboard warm request failed', { path, error: err.message })); + req.setTimeout(120000, () => req.destroy()); +} + +function warmLeaderboards() { + let seasons; + try { seasons = seasonsUtil.getSeasons(); } catch (e) { + return log.warn('Warmer: getSeasons failed', { error: e.message }); + } + const entries = Object.entries(seasons) + .map(([name, r]) => ({ name, start: r.start, end: r.end, status: r.status })) + .sort((a, b) => a.start - b.start); + const current = entries.filter(s => s.status === 'in_progress'); + const completed = entries.filter(s => s.status === 'completed'); + const lastCompleted = completed.length ? [completed[completed.length - 1]] : []; + const targets = [...current, ...lastCompleted]; + for (const s of targets) { + warmOneAggregate('/api/leaderboard/players', s.start, s.end); + warmOneAggregate('/api/leaderboard/vehicles', s.start, s.end); + warmOneAggregate('/api/leaderboard/squadrons', s.start, s.end); + } + log.info('Leaderboard warmer tick', { windows: targets.map(t => t.name) }); +} + // Best-effort sync lookup of a squadron name → clan_id using the warm cache. // Returns null if the cache isn't populated yet or the input doesn't resolve. // Callers fall back to text matching when this returns null. @@ -3484,22 +3570,9 @@ app.get('/api/leaderboard/players', (req, res) => { return { ...full, players: full.players.slice(0, limit), limit, returned: Math.min(limit, full.players.length) }; }; - const cacheKey = `leaderboard_players_${start_date || 'all'}_${end_date || 'all'}`; - const cached = getCachedResponse(cacheKey); - if (cached) { - log.info('Returning cached leaderboard response'); - return res.json(applyLimit(cached)); - } + const cacheKey = aggregateKey('leaderboard_players', dateFilters); - if (!dateFilters.hasFilter) { - return res.status(400).json({ - error: 'A date filter (start_date/end_date/season/week) is required for uncached all-time player leaderboard queries.', - errorCode: 'FILTER_REQUIRED' - }); - } - - // Dedup: if this exact query is already running, wait for it - dedup(cacheKey, () => new Promise((resolve, reject) => { + const compute = () => new Promise((resolve, reject) => { // Step 1: Pure aggregation — no nick lookups, single scan const statsQuery = ` SELECT @@ -3531,8 +3604,7 @@ app.get('/api/leaderboard/players', (req, res) => { heavyDb.all(statsQuery, queryParams, (err, statsRows) => { if (err) { log.error('Database error in player leaderboard aggregation', err); - reject(err); - return res.status(500).json({ error: 'Database error', errorCode: 'DB_PLAYER_LEADERBOARD_FAILED' }); + return reject(err); } log.info('Player leaderboard aggregation done', { rows: statsRows.length, ms: Date.now() - queryStart }); @@ -3589,9 +3661,7 @@ app.get('/api/leaderboard/players', (req, res) => { totalMs: Date.now() - queryStart }); - setCachedResponse(cacheKey, response); resolve(response); - res.json(applyLimit(response)); }; // Fallback for players not in any squadron — uses (UID, endtime_unix) index @@ -3614,10 +3684,12 @@ app.get('/api/leaderboard/players', (req, res) => { }); }); }); - })).catch(err => { - if (!res.headersSent) { - res.status(500).json({ error: 'Database error', errorCode: 'DB_PLAYER_LEADERBOARD_FAILED' }); - } + }); + + serveAggregateCached(cacheKey, dateFilters, compute, res, applyLimit, { + allowCompute: dateFilters.hasFilter, + filterError: 'A date filter (start_date/end_date/season/week) is required for uncached all-time player leaderboard queries.', + errorCode: 'DB_PLAYER_LEADERBOARD_FAILED', }); }); @@ -3641,21 +3713,9 @@ app.get('/api/leaderboard/vehicles', (req, res) => { return { ...full, vehicles: full.vehicles.slice(0, limit), limit, returned: Math.min(limit, full.vehicles.length) }; }; - const cacheKey = `leaderboard_vehicles_${vehicle || 'all'}_${start_date || 'all'}_${end_date || 'all'}`; - const cached = getCachedResponse(cacheKey); - if (cached) { - log.info('Returning cached vehicle leaderboard'); - return res.json(applyLimit(cached)); - } + const cacheKey = `leaderboard_vehicles_${vehicle || 'all'}_${dateFilters.startTimestamp ?? 'all'}_${dateFilters.endTimestamp ?? 'all'}`; - if (!vehicle && !dateFilters.hasFilter) { - return res.status(400).json({ - error: 'A date filter (start_date/end_date/season/week) or vehicle filter is required for uncached all-time vehicle leaderboard queries.', - errorCode: 'FILTER_REQUIRED' - }); - } - - dedup(cacheKey, () => new Promise((resolve, reject) => { + const compute = () => new Promise((resolve, reject) => { let statsQuery; let queryParams; @@ -3697,8 +3757,7 @@ app.get('/api/leaderboard/vehicles', (req, res) => { heavyDb.all(statsQuery, queryParams, (err, vehicleRows) => { if (err) { log.error('Database error in vehicle leaderboard', err); - reject(err); - return res.status(500).json({ error: 'Database error', errorCode: 'DB_VEHICLE_LEADERBOARD_FAILED' }); + return reject(err); } // Nick/squadron lookup from squadron_members cache (instant) @@ -3745,9 +3804,7 @@ app.get('/api/leaderboard/vehicles', (req, res) => { }; log.info('Vehicle leaderboard complete', { vehiclesReturned: vehicleRows.length, totalMs: Date.now() - queryStart }); - setCachedResponse(cacheKey, response); resolve(response); - res.json(applyLimit(response)); }; if (uncoveredUids.length > 0) { @@ -3768,10 +3825,12 @@ app.get('/api/leaderboard/vehicles', (req, res) => { }); }); }); - })).catch(err => { - if (!res.headersSent) { - res.status(500).json({ error: 'Database error', errorCode: 'DB_VEHICLE_LEADERBOARD_FAILED' }); - } + }); + + serveAggregateCached(cacheKey, dateFilters, compute, res, applyLimit, { + allowCompute: Boolean(vehicle) || dateFilters.hasFilter, + filterError: 'A date filter (start_date/end_date/season/week) or vehicle filter is required for uncached all-time vehicle leaderboard queries.', + errorCode: 'DB_VEHICLE_LEADERBOARD_FAILED', }); }); @@ -3796,14 +3855,9 @@ app.get('/api/leaderboard/squadrons', (req, res) => { }; }; - const cacheKey = `leaderboard_squadrons_${start_date || 'all'}_${end_date || 'all'}`; - const cached = getCachedResponse(cacheKey); - if (cached) { - log.info('Returning cached squadron leaderboard'); - return res.json(applyLimit(cached)); - } + const cacheKey = aggregateKey('leaderboard_squadrons', dateFilters); - dedup(cacheKey, () => new Promise((resolve, reject) => { + const compute = () => new Promise((resolve, reject) => { const schemaQuery = `PRAGMA table_info(player_games_hist)`; db.all(schemaQuery, (err, schema) => { @@ -3824,7 +3878,6 @@ app.get('/api/leaderboard/squadrons', (req, res) => { migration_needed: true, message: 'Squadron data not available - squadron_name column missing from database' }; - setCachedResponse(cacheKey, emptyResponse); return resolve(emptyResponse); } @@ -4057,22 +4110,17 @@ app.get('/api/leaderboard/squadrons', (req, res) => { squadronsWithPoints: squadronsArray.filter(s => s.points.has_points_data).length }); - setCachedResponse(cacheKey, response); resolve(response); }); }); }); }); }); - })).then(response => { - if (!res.headersSent) res.json(applyLimit(response)); - }).catch(err => { - if (res.headersSent) return; - if (err && err.status && err.body) { - return res.status(err.status).json(err.body); - } - log.error('Unhandled error in squadron leaderboard', err); - res.status(500).json({ error: 'Database error occurred', errorCode: 'DB_SQUADRON_LEADERBOARD_FAILED' }); + }); + + serveAggregateCached(cacheKey, dateFilters, compute, res, applyLimit, { + allowCompute: true, + errorCode: 'DB_SQUADRON_LEADERBOARD_FAILED', }); }); @@ -6033,6 +6081,11 @@ setTimeout(() => { }, 600000); // Every 10 minutes }, 150000); // Start after 2.5 min offset +// Warm hot leaderboard windows shortly after boot, then every 4 min. +// The first tick waits for the DB-ready grace so heavy scans don't fight startup. +setTimeout(warmLeaderboards, 20000); +setInterval(warmLeaderboards, 4 * 60 * 1000); + app.listen(PORT, () => { console.log(`SREBOT Player API server running on port ${PORT}`); console.log(`Health check: http://localhost:${PORT}/health`); diff --git a/start_server.sh b/start_server.sh new file mode 100755 index 0000000..c9c8be9 --- /dev/null +++ b/start_server.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Wrapper so libuv reads UV_THREADPOOL_SIZE from the OS environ block. +# PM2's env:/env_file don't propagate to child processes on this system. +export UV_THREADPOOL_SIZE=24 +exec node --max-old-space-size=6144 server.js diff --git a/web/server.js b/web/server.js index 1426c56..2871771 100644 --- a/web/server.js +++ b/web/server.js @@ -509,31 +509,30 @@ async function getCachedData(type) { return cache.data; } -// Initialize cache on startup — staggered to avoid overloading SQLite +// Initialize only lightweight caches on the primary worker. +// Player and vehicle leaderboards are too expensive to refresh in the background: +// recent production runs took 140s+ and kept the API CPU saturated after callers timed out. async function initializeCache() { - log.info('[CACHE] Initializing leaderboard cache (staggered)...'); - // Stats is lightest, do it first + if (!IS_PRIMARY_WORKER) return; + + log.info('[CACHE] Initializing lightweight leaderboard cache (staggered)...'); await updateCache('stats'); log.info('[CACHE] Stats cache ready'); - // Squadrons next await updateCache('squadrons'); log.info('[CACHE] Squadrons cache ready'); - // Players is heaviest - await updateCache('players'); - log.info('[CACHE] Players cache ready'); - // Vehicles last - await updateCache('vehicles'); - log.info('[CACHE] Vehicles cache ready — all caches populated!'); } -// Auto-refresh caches staggered to avoid hammering the API all at once -const cacheTypes = ['players', 'vehicles', 'stats', 'squadrons']; -cacheTypes.forEach((type, i) => { - setInterval(async () => { - log.debug(`[CACHE] Auto-refreshing ${type} cache...`); - await updateCache(type).catch(log.error); - }, CACHE_DURATION + i * 15000); // stagger each by 15s -}); +// Auto-refresh only lightweight caches from the primary worker. Heavy caches are +// refreshed lazily by request so background work cannot pin SQLite indefinitely. +if (IS_PRIMARY_WORKER) { + const cacheTypes = ['stats', 'squadrons']; + cacheTypes.forEach((type, i) => { + setInterval(async () => { + log.debug(`[CACHE] Auto-refreshing ${type} cache...`); + await updateCache(type).catch(log.error); + }, CACHE_DURATION + i * 15000); + }); +} // Log search cache statistics every 10 minutes setInterval(() => {