From 7f1e6d0beff614dea19277022fc93ef476527110 Mon Sep 17 00:00:00 2001 From: Heidi Date: Sat, 20 Jun 2026 00:20:48 +0100 Subject: [PATCH] ai generated solutions to our ai generated problems --- README.md | 15 ++-- example.env | 3 +- frontend/public/data/README.md | 6 +- frontend/src/App.jsx | 9 ++- server.cjs | 140 ++++++++++++++++++++++++++++----- 5 files changed, 139 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index c61834f..c38f48c 100644 --- a/README.md +++ b/README.md @@ -86,12 +86,12 @@ Vehicle icon PNGs are served statically at `/vehicle-icons` from `VEHICLE_ICONS_ The proxy blocks cross-origin/API-navigation requests, strips CORS headers from the upstream response, rate limits callers, and caches successful GET responses. -Public TSS reads are also written to a bounded JSON snapshot cache and served at -both their `/api/tss/*` route and matching `/data/*` path. Fresh snapshots return -without touching the backend; stale snapshots are served immediately while the -server refreshes them in the background. The frontend uses `/api/tss/*` by -default so it can hit this server-side cache directly; set `VITE_STATIC_DATA=true` -only if you explicitly want the client to try `/data/*` first. All responses +Public TSS reads are written to a bounded JSON snapshot cache and served at both +their `/api/tss/*` route and matching `/data/*` path. The frontend uses `/data/*` +by default for public pages. Fresh snapshots return without touching the backend; +stale snapshots are served immediately while the server refreshes them in the +background. Missing `/data/*` snapshots are filled from the matching upstream API +with a short timeout, then written atomically for future requests. All responses ship `X-Content-Type-Options`, `X-Frame-Options: DENY`, `Referrer-Policy`, `Permissions-Policy`, `Cross-Origin-Opener-Policy`, `Cross-Origin-Resource-Policy`, HSTS (over HTTPS), and HTML responses include a Content Security Policy that @@ -118,7 +118,8 @@ PUBLIC_DATA_CACHE_DIR=~/tsswebstorage/public-data PUBLIC_DATA_CACHE_FRESH_MS=300000 PUBLIC_DATA_CACHE_STALE_MS=86400000 PUBLIC_DATA_PREWARM_INTERVAL_MS=300000 -VITE_STATIC_DATA=false +PUBLIC_DATA_COLD_TIMEOUT_MS=8000 +VITE_STATIC_DATA=true API_RATE_LIMIT_WINDOW_MS=60000 API_RATE_LIMIT_MAX=120 ``` diff --git a/example.env b/example.env index e866079..b5c6ae8 100644 --- a/example.env +++ b/example.env @@ -37,6 +37,7 @@ PUBLIC_DATA_CACHE_DIR=~/tsswebstorage/public-data PUBLIC_DATA_CACHE_FRESH_MS=300000 PUBLIC_DATA_CACHE_STALE_MS=86400000 PUBLIC_DATA_PREWARM_INTERVAL_MS=300000 +PUBLIC_DATA_COLD_TIMEOUT_MS=8000 API_RATE_LIMIT_WINDOW_MS=60000 API_RATE_LIMIT_MAX=120 @@ -61,4 +62,4 @@ DISCORD_INCLUDE_PATCH=true # TURNSTILE_SECRET_KEY is the server-only secret used to call the Siteverify endpoint. VITE_TURNSTILE_SITE_KEY= TURNSTILE_SECRET_KEY= -VITE_STATIC_DATA=false +VITE_STATIC_DATA=true diff --git a/frontend/public/data/README.md b/frontend/public/data/README.md index 5dd8a4d..a750291 100644 --- a/frontend/public/data/README.md +++ b/frontend/public/data/README.md @@ -1,8 +1,8 @@ # Static Public Data -The frontend can try these JSON snapshots before falling back to the live API -when `VITE_STATIC_DATA=true`. By default, the app requests `/api/tss/*` directly -and lets the web server answer from the same public data cache. +The frontend tries these JSON snapshots before falling back to the live API by +default. The web server serves `/data/*` from the public data cache and fills +missing snapshots from the matching `/api/tss/*` route with a short timeout. - `/data/leaderboard-teams.json` - `/data/leaderboard-players.json` diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 49b751c..81626fe 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -51,7 +51,7 @@ const siteVersion = '1.0.1' const turnstileSiteKey = import.meta.env.VITE_TURNSTILE_SITE_KEY || '' const staticDataBase = (import.meta.env.VITE_STATIC_DATA_BASE || '/data').replace(/\/+$/, '') -const staticDataEnabled = String(import.meta.env.VITE_STATIC_DATA || 'false').toLowerCase() === 'true' +const staticDataEnabled = String(import.meta.env.VITE_STATIC_DATA || 'true').toLowerCase() !== 'false' const missingStaticDataPaths = new Set() const defaultAnalyticsPreferences = { @@ -126,8 +126,11 @@ async function fetchPublicJson(source, signal) { return await fetchJson(source.staticPath, signal) } catch (error) { if (signal?.aborted || error?.name === 'AbortError') throw error - if (error?.status === 404 || error?.contentType) missingStaticDataPaths.add(source.staticPath) - return fetchJson(source.apiPath, signal) + if (error?.status === 404 || error?.contentType) { + missingStaticDataPaths.add(source.staticPath) + return fetchJson(source.apiPath, signal) + } + throw error } } diff --git a/server.cjs b/server.cjs index a704b19..732efac 100644 --- a/server.cjs +++ b/server.cjs @@ -54,6 +54,7 @@ const PUBLIC_DATA_CACHE_DIR = path.resolve( const PUBLIC_DATA_CACHE_FRESH_MS = Number(process.env.PUBLIC_DATA_CACHE_FRESH_MS || 5 * 60 * 1000) const PUBLIC_DATA_CACHE_STALE_MS = Number(process.env.PUBLIC_DATA_CACHE_STALE_MS || 24 * 60 * 60 * 1000) const PUBLIC_DATA_PREWARM_INTERVAL_MS = Number(process.env.PUBLIC_DATA_PREWARM_INTERVAL_MS || PUBLIC_DATA_CACHE_FRESH_MS) +const PUBLIC_DATA_COLD_TIMEOUT_MS = Number(process.env.PUBLIC_DATA_COLD_TIMEOUT_MS || 8000) const PUBLIC_DATA_CACHE_MAX_AGE_SECONDS = Math.max(0, Math.floor(PUBLIC_DATA_CACHE_FRESH_MS / 1000)) const PUBLIC_DATA_STALE_REVALIDATE_SECONDS = Math.max(0, Math.floor(PUBLIC_DATA_CACHE_STALE_MS / 1000)) const API_RATE_LIMIT_WINDOW_MS = Number(process.env.API_RATE_LIMIT_WINDOW_MS || 60000) @@ -220,6 +221,14 @@ function publicDataKey(value) { return encodeURIComponent(String(value || '').trim()).replace(/%/g, '~') } +function publicDataValue(value) { + try { + return decodeURIComponent(String(value || '').replace(/~/g, '%')) + } catch { + return '' + } +} + function publicDataCachePathForUrl(requestUrl) { const pathname = requestUrl.pathname const params = requestUrl.searchParams @@ -269,6 +278,45 @@ function publicDataCachePathForUrl(requestUrl) { return null } +function publicDataApiPathForRelative(relativePath) { + if (relativePath === 'leaderboard-teams.json') return '/api/tss/leaderboard/teams?limit=100' + if (relativePath === 'leaderboard-players.json') return '/api/tss/leaderboard/players?limit=100' + if (relativePath === 'home-teams.json') return '/api/tss/leaderboard/teams?limit=4' + if (relativePath === 'recent-games.json') return '/api/tss/games/recent?limit=50' + + let match = relativePath.match(/^teams\/(.+)\.games\.json$/) + if (match) { + const team = publicDataValue(match[1]) + return team ? `/api/tss/teams/${encodeURIComponent(team)}/games` : '' + } + + match = relativePath.match(/^teams\/(.+)\.json$/) + if (match) { + const team = publicDataValue(match[1]) + return team ? `/api/tss/teams/${encodeURIComponent(team)}` : '' + } + + match = relativePath.match(/^players\/(.+)\.json$/) + if (match) { + const uid = publicDataValue(match[1]) + return /^[0-9]{1,32}$/.test(uid) ? `/api/tss/player/${encodeURIComponent(uid)}` : '' + } + + match = relativePath.match(/^games\/(.+)\.logs\.json$/) + if (match) { + const gameId = publicDataValue(match[1]) + return /^[A-Za-z0-9_-]{1,96}$/.test(gameId) ? `/api/tss/games/${encodeURIComponent(gameId)}/logs` : '' + } + + match = relativePath.match(/^games\/(.+)\.json$/) + if (match) { + const gameId = publicDataValue(match[1]) + return /^[A-Za-z0-9_-]{1,96}$/.test(gameId) ? `/api/tss/games/${encodeURIComponent(gameId)}` : '' + } + + return '' +} + function safePublicDataPath(requestPath) { let decoded = '/' try { @@ -286,6 +334,14 @@ function safePublicDataPath(requestPath) { return filePath } +function publicDataRelativePath(requestPath) { + try { + return decodeURIComponent(requestPath).replace(/^\/data\/?/, '') + } catch { + return '' + } +} + function sendPublicDataFile(req, res, filePath, status = 200, extraHeaders = {}) { fs.readFile(filePath, (error, data) => { if (error) { @@ -301,6 +357,40 @@ function sendPublicDataFile(req, res, filePath, status = 200, extraHeaders = {}) }) } +async function servePublicData(req, res, pathname) { + const filePath = safePublicDataPath(pathname) + const relativePath = publicDataRelativePath(pathname) + if (!filePath || !relativePath) { + sendJson(res, 400, { error: 'Bad request' }) + return + } + + const current = cachedPublicData(filePath) + if (current?.fresh) { + sendPublicDataFile(req, res, filePath, 200, { 'x-tssbot-cache': 'data-hit' }) + return + } + + const apiPath = publicDataApiPathForRelative(relativePath) + if (!apiPath) { + sendJson(res, 404, { error: 'Snapshot not found' }) + return + } + + if (current) { + refreshPublicData(filePath, new URL(apiPath, API_UPSTREAM)) + sendPublicDataFile(req, res, filePath, 200, { 'x-tssbot-cache': 'data-stale' }) + return + } + + try { + await fillPublicData(filePath, new URL(apiPath, API_UPSTREAM), PUBLIC_DATA_COLD_TIMEOUT_MS) + sendPublicDataFile(req, res, filePath, 200, { 'x-tssbot-cache': 'data-filled' }) + } catch (error) { + sendJson(res, 504, { error: 'Snapshot unavailable', detail: error.message }) + } +} + function cachedPublicData(filePath) { if (!filePath) return null @@ -322,16 +412,24 @@ function writePublicDataFile(filePath, body) { fs.renameSync(tmpPath, filePath) } -function refreshPublicData(filePath, target) { - if (!filePath || publicDataRefreshes.has(filePath)) return +async function fillPublicData(filePath, target, timeoutMs = SERVER_REQUEST_TIMEOUT_MS) { + if (!filePath) return false + if (publicDataRefreshes.has(filePath)) return false publicDataRefreshes.add(filePath) - fetchUpstreamJsonBuffer(target) - .then((body) => writePublicDataFile(filePath, body)) - .catch((error) => { - console.warn(`public data refresh failed for ${target.pathname}: ${error.message}`) - }) - .finally(() => publicDataRefreshes.delete(filePath)) + try { + const body = await fetchUpstreamJsonBuffer(target, timeoutMs) + writePublicDataFile(filePath, body) + return true + } finally { + publicDataRefreshes.delete(filePath) + } +} + +function refreshPublicData(filePath, target) { + fillPublicData(filePath, target).catch((error) => { + console.warn(`public data refresh failed for ${target.pathname}: ${error.message}`) + }) } function publicDataPrewarmTargets() { @@ -351,21 +449,26 @@ function publicDataPrewarmTargets() { } function prewarmPublicDataCache() { + const jobs = [] for (const { filePath, target } of publicDataPrewarmTargets()) { const current = cachedPublicData(filePath) if (current?.fresh) continue - refreshPublicData(filePath, target) + jobs.push(fillPublicData(filePath, target, PUBLIC_DATA_COLD_TIMEOUT_MS)) } + return Promise.allSettled(jobs) } function startPublicDataPrewarmer() { fs.mkdirSync(PUBLIC_DATA_CACHE_DIR, { recursive: true }) - prewarmPublicDataCache() - publicDataPrewarmTimer = setInterval(prewarmPublicDataCache, PUBLIC_DATA_PREWARM_INTERVAL_MS) + publicDataPrewarmTimer = setInterval(() => { + prewarmPublicDataCache().catch((error) => { + console.warn(`public data prewarm failed: ${error.message}`) + }) + }, PUBLIC_DATA_PREWARM_INTERVAL_MS) publicDataPrewarmTimer.unref?.() } -function fetchUpstreamJsonBuffer(target) { +function fetchUpstreamJsonBuffer(target, timeoutMs = SERVER_REQUEST_TIMEOUT_MS) { return new Promise((resolve, reject) => { const client = target.protocol === 'https:' ? https : http const chunks = [] @@ -401,7 +504,7 @@ function fetchUpstreamJsonBuffer(target) { }, ) - proxy.setTimeout(SERVER_REQUEST_TIMEOUT_MS, () => proxy.destroy(new Error('Upstream request timed out'))) + proxy.setTimeout(timeoutMs, () => proxy.destroy(new Error(`Upstream request timed out after ${timeoutMs}ms`))) proxy.on('error', reject) proxy.end() }) @@ -2687,12 +2790,7 @@ const server = http.createServer((req, res) => { } if (pathname.startsWith('/data/')) { - const filePath = safePublicDataPath(pathname) - if (!filePath) { - sendJson(res, 400, { error: 'Bad request' }) - return - } - sendPublicDataFile(req, res, filePath) + servePublicData(req, res, pathname) return } } @@ -2713,7 +2811,7 @@ const server = http.createServer((req, res) => { server.requestTimeout = SERVER_REQUEST_TIMEOUT_MS server.headersTimeout = SERVER_HEADERS_TIMEOUT_MS -server.listen(PORT, '0.0.0.0', () => { +server.listen(PORT, '0.0.0.0', async () => { console.log(`tssbot-web serving http://localhost:${PORT}`) console.log(`proxying API requests to ${API_UPSTREAM}`) if (RUN_BACKGROUND_JOBS) { @@ -2729,6 +2827,8 @@ server.listen(PORT, '0.0.0.0', () => { } if (RUN_BACKGROUND_JOBS) { startUptimeSampler() + fs.mkdirSync(PUBLIC_DATA_CACHE_DIR, { recursive: true }) + await prewarmPublicDataCache() startPublicDataPrewarmer() } process.send?.('ready')