From a60999a54e9f16a554b2932b31f9a636acd7c37b Mon Sep 17 00:00:00 2001 From: Heidi Date: Sat, 20 Jun 2026 00:05:10 +0100 Subject: [PATCH] ai generated solutions to our ai generated problems --- README.md | 11 ++- example.env | 4 + server.cjs | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 254 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e63d72..e5732d6 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,11 @@ Vehicle icon PNGs are served statically at `/vehicle-icons` from `VEHICLE_ICONS_ (populated at deploy from `SHARED/ICONS/VEHICLES`). The proxy blocks cross-origin/API-navigation requests, strips CORS headers from -the upstream response, rate limits callers, and caches successful GET responses -briefly so public page traffic does not hammer the upstream API. All responses +the upstream response, rate limits callers, and caches successful GET responses. +Public TSS reads are also written to a bounded JSON snapshot cache and served at +both their `/api/tss/*` route and matching `/data/*` path. Fresh snapshots return +without touching the backend; stale snapshots are served immediately while the +server refreshes them in the background. All responses ship `X-Content-Type-Options`, `X-Frame-Options: DENY`, `Referrer-Policy`, `Permissions-Policy`, `Cross-Origin-Opener-Policy`, `Cross-Origin-Resource-Policy`, HSTS (over HTTPS), and HTML responses include a Content Security Policy that @@ -109,6 +112,10 @@ Optional API protection tuning: ```sh API_CACHE_TTL_MS=15000 +PUBLIC_DATA_CACHE_DIR=~/tsswebstorage/public-data +PUBLIC_DATA_CACHE_FRESH_MS=60000 +PUBLIC_DATA_CACHE_STALE_MS=86400000 +PUBLIC_DATA_PREWARM_INTERVAL_MS=60000 API_RATE_LIMIT_WINDOW_MS=60000 API_RATE_LIMIT_MAX=120 ``` diff --git a/example.env b/example.env index 5252507..d14e602 100644 --- a/example.env +++ b/example.env @@ -33,6 +33,10 @@ ANALYTICS_RETENTION_DAYS=30 ANALYTICS_ACTIVE_WINDOW_SECONDS=75 API_CACHE_TTL_MS=15000 +PUBLIC_DATA_CACHE_DIR=~/tsswebstorage/public-data +PUBLIC_DATA_CACHE_FRESH_MS=60000 +PUBLIC_DATA_CACHE_STALE_MS=86400000 +PUBLIC_DATA_PREWARM_INTERVAL_MS=60000 API_RATE_LIMIT_WINDOW_MS=60000 API_RATE_LIMIT_MAX=120 diff --git a/server.cjs b/server.cjs index d95c07d..80416d5 100644 --- a/server.cjs +++ b/server.cjs @@ -48,6 +48,12 @@ const ANALYTICS_DATABASE_FILE = process.env.ANALYTICS_DATABASE_FILE || 'viewers. const ANALYTICS_RETENTION_DAYS = Number(process.env.ANALYTICS_RETENTION_DAYS || 30) const ANALYTICS_ACTIVE_WINDOW_SECONDS = Number(process.env.ANALYTICS_ACTIVE_WINDOW_SECONDS || 75) const API_CACHE_TTL_MS = Number(process.env.API_CACHE_TTL_MS || 15000) +const PUBLIC_DATA_CACHE_DIR = path.resolve( + expandHome(process.env.PUBLIC_DATA_CACHE_DIR || path.join(UPTIME_STORAGE_DIR, 'public-data')), +) +const PUBLIC_DATA_CACHE_FRESH_MS = Number(process.env.PUBLIC_DATA_CACHE_FRESH_MS || 60 * 1000) +const PUBLIC_DATA_CACHE_STALE_MS = Number(process.env.PUBLIC_DATA_CACHE_STALE_MS || 24 * 60 * 60 * 1000) +const PUBLIC_DATA_PREWARM_INTERVAL_MS = Number(process.env.PUBLIC_DATA_PREWARM_INTERVAL_MS || PUBLIC_DATA_CACHE_FRESH_MS) const API_RATE_LIMIT_WINDOW_MS = Number(process.env.API_RATE_LIMIT_WINDOW_MS || 60000) const API_RATE_LIMIT_MAX = Number(process.env.API_RATE_LIMIT_MAX || 120) const TURNSTILE_SECRET_KEY = process.env.TURNSTILE_SECRET_KEY || '' @@ -197,15 +203,208 @@ const jsonHeaders = { } const apiCache = new Map() +const publicDataRefreshes = new Set() const rateLimits = new Map() let uptimeDb = null let analyticsDb = null let latestUptimeSnapshot = null +let publicDataPrewarmTimer = null function sendJson(res, status, body, headers = {}) { send(res, status, JSON.stringify(body), { ...jsonHeaders, ...headers }) } +function publicDataKey(value) { + return encodeURIComponent(String(value || '').trim()).replace(/%/g, '~') +} + +function publicDataCachePathForUrl(requestUrl) { + const pathname = requestUrl.pathname + const params = requestUrl.searchParams + + if (pathname === '/api/tss/leaderboard/teams') { + const limit = Number(params.get('limit') || 100) + if (limit === 4) return path.join(PUBLIC_DATA_CACHE_DIR, 'home-teams.json') + if (limit === 100) return path.join(PUBLIC_DATA_CACHE_DIR, 'leaderboard-teams.json') + return null + } + + if (pathname === '/api/tss/leaderboard/players') { + const limit = Number(params.get('limit') || 100) + return limit === 100 ? path.join(PUBLIC_DATA_CACHE_DIR, 'leaderboard-players.json') : null + } + + if (pathname === '/api/tss/games/recent') { + const limit = Number(params.get('limit') || 50) + return limit === 50 ? path.join(PUBLIC_DATA_CACHE_DIR, 'recent-games.json') : null + } + + let match = pathname.match(/^\/api\/tss\/teams\/([^/]+)$/) + if (match && ![...params.keys()].length) { + return path.join(PUBLIC_DATA_CACHE_DIR, 'teams', `${publicDataKey(decodeURIComponent(match[1]))}.json`) + } + + match = pathname.match(/^\/api\/tss\/teams\/([^/]+)\/games$/) + if (match && ![...params.keys()].length) { + return path.join(PUBLIC_DATA_CACHE_DIR, 'teams', `${publicDataKey(decodeURIComponent(match[1]))}.games.json`) + } + + match = pathname.match(/^\/api\/tss\/player\/([0-9]{1,32})$/) + if (match && ![...params.keys()].length) { + return path.join(PUBLIC_DATA_CACHE_DIR, 'players', `${publicDataKey(match[1])}.json`) + } + + match = pathname.match(/^\/api\/tss\/games\/([A-Za-z0-9_-]{1,96})$/) + if (match && (params.get('lang') || 'en') === 'en' && [...params.keys()].every((key) => key === 'lang')) { + return path.join(PUBLIC_DATA_CACHE_DIR, 'games', `${publicDataKey(match[1])}.json`) + } + + match = pathname.match(/^\/api\/tss\/games\/([A-Za-z0-9_-]{1,96})\/logs$/) + if (match && ![...params.keys()].length) { + return path.join(PUBLIC_DATA_CACHE_DIR, 'games', `${publicDataKey(match[1])}.logs.json`) + } + + return null +} + +function safePublicDataPath(requestPath) { + let decoded = '/' + try { + decoded = decodeURIComponent(requestPath) + } catch { + return null + } + + const relative = decoded.replace(/^\/data\/?/, '') + if (!relative || relative.includes('\0')) return null + + const filePath = path.resolve(PUBLIC_DATA_CACHE_DIR, relative) + const relativeToCache = path.relative(PUBLIC_DATA_CACHE_DIR, filePath) + if (relativeToCache.startsWith('..') || path.isAbsolute(relativeToCache)) return null + return filePath +} + +function sendPublicDataFile(req, res, filePath, status = 200, extraHeaders = {}) { + fs.readFile(filePath, (error, data) => { + if (error) { + sendJson(res, 404, { error: 'Snapshot not found' }) + return + } + + send(res, status, data, { + 'content-type': 'application/json; charset=utf-8', + 'cache-control': 'public, max-age=30, stale-while-revalidate=300', + ...extraHeaders, + }) + }) +} + +function cachedPublicData(filePath) { + if (!filePath) return null + + try { + const stat = fs.statSync(filePath) + if (!stat.isFile()) return null + const age = Date.now() - stat.mtimeMs + if (age > PUBLIC_DATA_CACHE_STALE_MS) return null + return { filePath, age, fresh: age <= PUBLIC_DATA_CACHE_FRESH_MS } + } catch { + return null + } +} + +function writePublicDataFile(filePath, body) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }) + const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp` + fs.writeFileSync(tmpPath, body) + fs.renameSync(tmpPath, filePath) +} + +function refreshPublicData(filePath, target) { + if (!filePath || publicDataRefreshes.has(filePath)) return + publicDataRefreshes.add(filePath) + + fetchUpstreamJsonBuffer(target) + .then((body) => writePublicDataFile(filePath, body)) + .catch((error) => { + console.warn(`public data refresh failed for ${target.pathname}: ${error.message}`) + }) + .finally(() => publicDataRefreshes.delete(filePath)) +} + +function publicDataPrewarmTargets() { + return [ + '/api/tss/leaderboard/teams?limit=100', + '/api/tss/leaderboard/players?limit=100', + '/api/tss/leaderboard/teams?limit=4', + '/api/tss/games/recent?limit=50', + ].map((requestPath) => { + const requestUrl = new URL(requestPath, 'http://localhost') + return { + requestUrl, + target: new URL(requestPath, API_UPSTREAM), + filePath: publicDataCachePathForUrl(requestUrl), + } + }).filter((entry) => entry.filePath) +} + +function prewarmPublicDataCache() { + for (const { filePath, target } of publicDataPrewarmTargets()) { + const current = cachedPublicData(filePath) + if (current?.fresh) continue + refreshPublicData(filePath, target) + } +} + +function startPublicDataPrewarmer() { + fs.mkdirSync(PUBLIC_DATA_CACHE_DIR, { recursive: true }) + prewarmPublicDataCache() + publicDataPrewarmTimer = setInterval(prewarmPublicDataCache, PUBLIC_DATA_PREWARM_INTERVAL_MS) + publicDataPrewarmTimer.unref?.() +} + +function fetchUpstreamJsonBuffer(target) { + return new Promise((resolve, reject) => { + const client = target.protocol === 'https:' ? https : http + const chunks = [] + let bytes = 0 + const proxy = client.request( + target, + { + method: 'GET', + headers: { + accept: 'application/json', + host: target.host, + 'user-agent': 'tssbot-web-cache', + }, + }, + (proxyRes) => { + const status = proxyRes.statusCode || 502 + const contentType = String(proxyRes.headers['content-type'] || '') + if (status < 200 || status >= 300 || !contentType.includes('application/json')) { + proxyRes.resume() + reject(new Error(`Upstream returned ${status}`)) + return + } + + proxyRes.on('data', (chunk) => { + bytes += chunk.length + if (bytes > MAX_UPSTREAM_BODY_BYTES) { + proxy.destroy(new Error('Upstream response too large')) + return + } + chunks.push(chunk) + }) + proxyRes.on('end', () => resolve(Buffer.concat(chunks))) + }, + ) + + proxy.setTimeout(SERVER_REQUEST_TIMEOUT_MS, () => proxy.destroy(new Error('Upstream request timed out'))) + proxy.on('error', reject) + proxy.end() + }) +} + function requestJson(url, timeoutMs = 10000) { return new Promise((resolve, reject) => { const target = new URL(url) @@ -1713,6 +1912,17 @@ function proxyRequest(req, res) { return sendJson(res, 404, { error: 'API route not found' }) } + const requestUrl = new URL(req.url, `http://${req.headers.host || 'localhost'}`) + const publicDataFile = req.method === 'GET' ? publicDataCachePathForUrl(requestUrl) : null + const publicData = cachedPublicData(publicDataFile) + if (publicData?.fresh) { + return sendPublicDataFile(req, res, publicData.filePath, 200, { 'x-tssbot-cache': 'public-data-hit' }) + } + if (publicData) { + refreshPublicData(publicData.filePath, target) + return sendPublicDataFile(req, res, publicData.filePath, 200, { 'x-tssbot-cache': 'public-data-stale' }) + } + const cacheKey = req.method === 'GET' ? target.toString() : '' const cached = cacheKey ? apiCache.get(cacheKey) : null if (cached && cached.expiresAt > Date.now()) { @@ -1732,10 +1942,13 @@ function proxyRequest(req, res) { }, }, (proxyRes) => { + const statusCode = proxyRes.statusCode || 502 + const contentType = String(proxyRes.headers['content-type'] || '') + const shouldCacheJson = statusCode >= 200 && statusCode < 300 && contentType.includes('application/json') const headers = { ...proxyRes.headers, ...securityHeaders(req), - 'cache-control': 'private, max-age=15', + 'cache-control': publicDataFile ? 'public, max-age=30, stale-while-revalidate=300' : 'private, max-age=15', } delete headers['access-control-allow-origin'] @@ -1744,7 +1957,7 @@ function proxyRequest(req, res) { delete headers['access-control-allow-headers'] delete headers['access-control-expose-headers'] - res.writeHead(proxyRes.statusCode || 502, headers) + res.writeHead(statusCode, headers) proxyRes.on('data', (chunk) => { proxiedBytes += chunk.length @@ -1753,18 +1966,26 @@ function proxyRequest(req, res) { res.destroy() return } - if (cacheKey && (proxyRes.statusCode || 0) >= 200 && (proxyRes.statusCode || 0) < 300) { + if (cacheKey && shouldCacheJson) { responseChunks.push(chunk) } }) proxyRes.on('end', () => { if (cacheKey && responseChunks.length) { + const body = Buffer.concat(responseChunks) apiCache.set(cacheKey, { - body: Buffer.concat(responseChunks), + body, headers, expiresAt: Date.now() + API_CACHE_TTL_MS, }) + if (publicDataFile) { + try { + writePublicDataFile(publicDataFile, body) + } catch (error) { + console.warn(`could not write public data cache for ${requestUrl.pathname}: ${error.message}`) + } + } } }) @@ -2459,6 +2680,16 @@ const server = http.createServer((req, res) => { serveReplayMinimap(req, res) return } + + if (pathname.startsWith('/data/')) { + const filePath = safePublicDataPath(pathname) + if (!filePath) { + sendJson(res, 400, { error: 'Bad request' }) + return + } + sendPublicDataFile(req, res, filePath) + return + } } if (req.method === 'GET' && req.url.startsWith('/vehicle-icons/')) { @@ -2487,10 +2718,14 @@ server.listen(PORT, '0.0.0.0', () => { } console.log(`storing uptime snapshots in ${path.join(uptimeStoragePath(), UPTIME_DATABASE_FILE)}`) console.log(`storing viewer analytics in ${path.join(uptimeStoragePath(), ANALYTICS_DATABASE_FILE)}`) + console.log(`storing public data cache in ${PUBLIC_DATA_CACHE_DIR}`) if (!TURNSTILE_SECRET_KEY) { console.warn('TURNSTILE_SECRET_KEY is not set — Turnstile verification is disabled and gated endpoints will accept any request') } - if (RUN_BACKGROUND_JOBS) startUptimeSampler() + if (RUN_BACKGROUND_JOBS) { + startUptimeSampler() + startPublicDataPrewarmer() + } process.send?.('ready') }) @@ -2511,6 +2746,7 @@ function shutdown() { shuttingDown = true if (uptimeSamplerTimer) clearInterval(uptimeSamplerTimer) + if (publicDataPrewarmTimer) clearInterval(publicDataPrewarmTimer) server.close(() => { closeDatabase(uptimeDb, 'uptime')