ai generated solutions to our ai generated problems

README.md

@@ -85,8 +85,11 @@ Vehicle icon PNGs are served statically at `/vehicle-icons` from `VEHICLE_ICONS_
 (populated at deploy from `SHARED/ICONS/VEHICLES`).
 
 The proxy blocks cross-origin/API-navigation requests, strips CORS headers from
-the upstream response, rate limits callers, and caches successful GET responses
+the upstream response, rate limits callers, and caches successful GET responses.
-briefly so public page traffic does not hammer the upstream API. All responses
+Public TSS reads are also written to a bounded JSON snapshot cache and served at
+both their `/api/tss/*` route and matching `/data/*` path. Fresh snapshots return
+without touching the backend; stale snapshots are served immediately while the
+server refreshes them in the background. All responses
 ship `X-Content-Type-Options`, `X-Frame-Options: DENY`, `Referrer-Policy`,
 `Permissions-Policy`, `Cross-Origin-Opener-Policy`, `Cross-Origin-Resource-Policy`,
 HSTS (over HTTPS), and HTML responses include a Content Security Policy that
@@ -109,6 +112,10 @@ Optional API protection tuning:
 
 ```sh
 API_CACHE_TTL_MS=15000
+PUBLIC_DATA_CACHE_DIR=~/tsswebstorage/public-data
+PUBLIC_DATA_CACHE_FRESH_MS=60000
+PUBLIC_DATA_CACHE_STALE_MS=86400000
+PUBLIC_DATA_PREWARM_INTERVAL_MS=60000
 API_RATE_LIMIT_WINDOW_MS=60000
 API_RATE_LIMIT_MAX=120
 ```

example.env

@@ -33,6 +33,10 @@ ANALYTICS_RETENTION_DAYS=30
 ANALYTICS_ACTIVE_WINDOW_SECONDS=75
 
 API_CACHE_TTL_MS=15000
+PUBLIC_DATA_CACHE_DIR=~/tsswebstorage/public-data
+PUBLIC_DATA_CACHE_FRESH_MS=60000
+PUBLIC_DATA_CACHE_STALE_MS=86400000
+PUBLIC_DATA_PREWARM_INTERVAL_MS=60000
 API_RATE_LIMIT_WINDOW_MS=60000
 API_RATE_LIMIT_MAX=120
 

server.cjs

@@ -48,6 +48,12 @@ const ANALYTICS_DATABASE_FILE = process.env.ANALYTICS_DATABASE_FILE || 'viewers.
 const ANALYTICS_RETENTION_DAYS = Number(process.env.ANALYTICS_RETENTION_DAYS || 30)
 const ANALYTICS_ACTIVE_WINDOW_SECONDS = Number(process.env.ANALYTICS_ACTIVE_WINDOW_SECONDS || 75)
 const API_CACHE_TTL_MS = Number(process.env.API_CACHE_TTL_MS || 15000)
+const PUBLIC_DATA_CACHE_DIR = path.resolve(
+  expandHome(process.env.PUBLIC_DATA_CACHE_DIR || path.join(UPTIME_STORAGE_DIR, 'public-data')),
+)
+const PUBLIC_DATA_CACHE_FRESH_MS = Number(process.env.PUBLIC_DATA_CACHE_FRESH_MS || 60 * 1000)
+const PUBLIC_DATA_CACHE_STALE_MS = Number(process.env.PUBLIC_DATA_CACHE_STALE_MS || 24 * 60 * 60 * 1000)
+const PUBLIC_DATA_PREWARM_INTERVAL_MS = Number(process.env.PUBLIC_DATA_PREWARM_INTERVAL_MS || PUBLIC_DATA_CACHE_FRESH_MS)
 const API_RATE_LIMIT_WINDOW_MS = Number(process.env.API_RATE_LIMIT_WINDOW_MS || 60000)
 const API_RATE_LIMIT_MAX = Number(process.env.API_RATE_LIMIT_MAX || 120)
 const TURNSTILE_SECRET_KEY = process.env.TURNSTILE_SECRET_KEY || ''
@@ -197,15 +203,208 @@ const jsonHeaders = {
 }
 
 const apiCache = new Map()
+const publicDataRefreshes = new Set()
 const rateLimits = new Map()
 let uptimeDb = null
 let analyticsDb = null
 let latestUptimeSnapshot = null
+let publicDataPrewarmTimer = null
 
 function sendJson(res, status, body, headers = {}) {
   send(res, status, JSON.stringify(body), { ...jsonHeaders, ...headers })
 }
 
+function publicDataKey(value) {
+  return encodeURIComponent(String(value || '').trim()).replace(/%/g, '~')
+}
+
+function publicDataCachePathForUrl(requestUrl) {
+  const pathname = requestUrl.pathname
+  const params = requestUrl.searchParams
+
+  if (pathname === '/api/tss/leaderboard/teams') {
+    const limit = Number(params.get('limit') || 100)
+    if (limit === 4) return path.join(PUBLIC_DATA_CACHE_DIR, 'home-teams.json')
+    if (limit === 100) return path.join(PUBLIC_DATA_CACHE_DIR, 'leaderboard-teams.json')
+    return null
+  }
+
+  if (pathname === '/api/tss/leaderboard/players') {
+    const limit = Number(params.get('limit') || 100)
+    return limit === 100 ? path.join(PUBLIC_DATA_CACHE_DIR, 'leaderboard-players.json') : null
+  }
+
+  if (pathname === '/api/tss/games/recent') {
+    const limit = Number(params.get('limit') || 50)
+    return limit === 50 ? path.join(PUBLIC_DATA_CACHE_DIR, 'recent-games.json') : null
+  }
+
+  let match = pathname.match(/^\/api\/tss\/teams\/([^/]+)$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'teams', `${publicDataKey(decodeURIComponent(match[1]))}.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/teams\/([^/]+)\/games$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'teams', `${publicDataKey(decodeURIComponent(match[1]))}.games.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/player\/([0-9]{1,32})$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'players', `${publicDataKey(match[1])}.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/games\/([A-Za-z0-9_-]{1,96})$/)
+  if (match && (params.get('lang') || 'en') === 'en' && [...params.keys()].every((key) => key === 'lang')) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'games', `${publicDataKey(match[1])}.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/games\/([A-Za-z0-9_-]{1,96})\/logs$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'games', `${publicDataKey(match[1])}.logs.json`)
+  }
+
+  return null
+}
+
+function safePublicDataPath(requestPath) {
+  let decoded = '/'
+  try {
+    decoded = decodeURIComponent(requestPath)
+  } catch {
+    return null
+  }
+
+  const relative = decoded.replace(/^\/data\/?/, '')
+  if (!relative || relative.includes('\0')) return null
+
+  const filePath = path.resolve(PUBLIC_DATA_CACHE_DIR, relative)
+  const relativeToCache = path.relative(PUBLIC_DATA_CACHE_DIR, filePath)
+  if (relativeToCache.startsWith('..') || path.isAbsolute(relativeToCache)) return null
+  return filePath
+}
+
+function sendPublicDataFile(req, res, filePath, status = 200, extraHeaders = {}) {
+  fs.readFile(filePath, (error, data) => {
+    if (error) {
+      sendJson(res, 404, { error: 'Snapshot not found' })
+      return
+    }
+
+    send(res, status, data, {
+      'content-type': 'application/json; charset=utf-8',
+      'cache-control': 'public, max-age=30, stale-while-revalidate=300',
+      ...extraHeaders,
+    })
+  })
+}
+
+function cachedPublicData(filePath) {
+  if (!filePath) return null
+
+  try {
+    const stat = fs.statSync(filePath)
+    if (!stat.isFile()) return null
+    const age = Date.now() - stat.mtimeMs
+    if (age > PUBLIC_DATA_CACHE_STALE_MS) return null
+    return { filePath, age, fresh: age <= PUBLIC_DATA_CACHE_FRESH_MS }
+  } catch {
+    return null
+  }
+}
+
+function writePublicDataFile(filePath, body) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true })
+  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`
+  fs.writeFileSync(tmpPath, body)
+  fs.renameSync(tmpPath, filePath)
+}
+
+function refreshPublicData(filePath, target) {
+  if (!filePath || publicDataRefreshes.has(filePath)) return
+  publicDataRefreshes.add(filePath)
+
+  fetchUpstreamJsonBuffer(target)
+    .then((body) => writePublicDataFile(filePath, body))
+    .catch((error) => {
+      console.warn(`public data refresh failed for ${target.pathname}: ${error.message}`)
+    })
+    .finally(() => publicDataRefreshes.delete(filePath))
+}
+
+function publicDataPrewarmTargets() {
+  return [
+    '/api/tss/leaderboard/teams?limit=100',
+    '/api/tss/leaderboard/players?limit=100',
+    '/api/tss/leaderboard/teams?limit=4',
+    '/api/tss/games/recent?limit=50',
+  ].map((requestPath) => {
+    const requestUrl = new URL(requestPath, 'http://localhost')
+    return {
+      requestUrl,
+      target: new URL(requestPath, API_UPSTREAM),
+      filePath: publicDataCachePathForUrl(requestUrl),
+    }
+  }).filter((entry) => entry.filePath)
+}
+
+function prewarmPublicDataCache() {
+  for (const { filePath, target } of publicDataPrewarmTargets()) {
+    const current = cachedPublicData(filePath)
+    if (current?.fresh) continue
+    refreshPublicData(filePath, target)
+  }
+}
+
+function startPublicDataPrewarmer() {
+  fs.mkdirSync(PUBLIC_DATA_CACHE_DIR, { recursive: true })
+  prewarmPublicDataCache()
+  publicDataPrewarmTimer = setInterval(prewarmPublicDataCache, PUBLIC_DATA_PREWARM_INTERVAL_MS)
+  publicDataPrewarmTimer.unref?.()
+}
+
+function fetchUpstreamJsonBuffer(target) {
+  return new Promise((resolve, reject) => {
+    const client = target.protocol === 'https:' ? https : http
+    const chunks = []
+    let bytes = 0
+    const proxy = client.request(
+      target,
+      {
+        method: 'GET',
+        headers: {
+          accept: 'application/json',
+          host: target.host,
+          'user-agent': 'tssbot-web-cache',
+        },
+      },
+      (proxyRes) => {
+        const status = proxyRes.statusCode || 502
+        const contentType = String(proxyRes.headers['content-type'] || '')
+        if (status < 200 || status >= 300 || !contentType.includes('application/json')) {
+          proxyRes.resume()
+          reject(new Error(`Upstream returned ${status}`))
+          return
+        }
+
+        proxyRes.on('data', (chunk) => {
+          bytes += chunk.length
+          if (bytes > MAX_UPSTREAM_BODY_BYTES) {
+            proxy.destroy(new Error('Upstream response too large'))
+            return
+          }
+          chunks.push(chunk)
+        })
+        proxyRes.on('end', () => resolve(Buffer.concat(chunks)))
+      },
+    )
+
+    proxy.setTimeout(SERVER_REQUEST_TIMEOUT_MS, () => proxy.destroy(new Error('Upstream request timed out')))
+    proxy.on('error', reject)
+    proxy.end()
+  })
+}
+
 function requestJson(url, timeoutMs = 10000) {
   return new Promise((resolve, reject) => {
     const target = new URL(url)
@@ -1713,6 +1912,17 @@ function proxyRequest(req, res) {
     return sendJson(res, 404, { error: 'API route not found' })
   }
 
+  const requestUrl = new URL(req.url, `http://${req.headers.host || 'localhost'}`)
+  const publicDataFile = req.method === 'GET' ? publicDataCachePathForUrl(requestUrl) : null
+  const publicData = cachedPublicData(publicDataFile)
+  if (publicData?.fresh) {
+    return sendPublicDataFile(req, res, publicData.filePath, 200, { 'x-tssbot-cache': 'public-data-hit' })
+  }
+  if (publicData) {
+    refreshPublicData(publicData.filePath, target)
+    return sendPublicDataFile(req, res, publicData.filePath, 200, { 'x-tssbot-cache': 'public-data-stale' })
+  }
+
   const cacheKey = req.method === 'GET' ? target.toString() : ''
   const cached = cacheKey ? apiCache.get(cacheKey) : null
   if (cached && cached.expiresAt > Date.now()) {
@@ -1732,10 +1942,13 @@ function proxyRequest(req, res) {
       },
     },
     (proxyRes) => {
+      const statusCode = proxyRes.statusCode || 502
+      const contentType = String(proxyRes.headers['content-type'] || '')
+      const shouldCacheJson = statusCode >= 200 && statusCode < 300 && contentType.includes('application/json')
       const headers = {
         ...proxyRes.headers,
         ...securityHeaders(req),
-        'cache-control': 'private, max-age=15',
+        'cache-control': publicDataFile ? 'public, max-age=30, stale-while-revalidate=300' : 'private, max-age=15',
       }
 
       delete headers['access-control-allow-origin']
@@ -1744,7 +1957,7 @@ function proxyRequest(req, res) {
       delete headers['access-control-allow-headers']
       delete headers['access-control-expose-headers']
 
-      res.writeHead(proxyRes.statusCode || 502, headers)
+      res.writeHead(statusCode, headers)
 
       proxyRes.on('data', (chunk) => {
         proxiedBytes += chunk.length
@@ -1753,18 +1966,26 @@ function proxyRequest(req, res) {
           res.destroy()
           return
         }
-        if (cacheKey && (proxyRes.statusCode || 0) >= 200 && (proxyRes.statusCode || 0) < 300) {
+        if (cacheKey && shouldCacheJson) {
           responseChunks.push(chunk)
         }
       })
 
       proxyRes.on('end', () => {
         if (cacheKey && responseChunks.length) {
+          const body = Buffer.concat(responseChunks)
           apiCache.set(cacheKey, {
-            body: Buffer.concat(responseChunks),
+            body,
             headers,
             expiresAt: Date.now() + API_CACHE_TTL_MS,
           })
+          if (publicDataFile) {
+            try {
+              writePublicDataFile(publicDataFile, body)
+            } catch (error) {
+              console.warn(`could not write public data cache for ${requestUrl.pathname}: ${error.message}`)
+            }
+          }
         }
       })
 
@@ -2459,6 +2680,16 @@ const server = http.createServer((req, res) => {
       serveReplayMinimap(req, res)
       return
     }
+
+    if (pathname.startsWith('/data/')) {
+      const filePath = safePublicDataPath(pathname)
+      if (!filePath) {
+        sendJson(res, 400, { error: 'Bad request' })
+        return
+      }
+      sendPublicDataFile(req, res, filePath)
+      return
+    }
   }
 
   if (req.method === 'GET' && req.url.startsWith('/vehicle-icons/')) {
@@ -2487,10 +2718,14 @@ server.listen(PORT, '0.0.0.0', () => {
   }
   console.log(`storing uptime snapshots in ${path.join(uptimeStoragePath(), UPTIME_DATABASE_FILE)}`)
   console.log(`storing viewer analytics in ${path.join(uptimeStoragePath(), ANALYTICS_DATABASE_FILE)}`)
+  console.log(`storing public data cache in ${PUBLIC_DATA_CACHE_DIR}`)
   if (!TURNSTILE_SECRET_KEY) {
     console.warn('TURNSTILE_SECRET_KEY is not set — Turnstile verification is disabled and gated endpoints will accept any request')
   }
-  if (RUN_BACKGROUND_JOBS) startUptimeSampler()
+  if (RUN_BACKGROUND_JOBS) {
+    startUptimeSampler()
+    startPublicDataPrewarmer()
+  }
   process.send?.('ready')
 })
 
@@ -2511,6 +2746,7 @@ function shutdown() {
   shuttingDown = true
 
   if (uptimeSamplerTimer) clearInterval(uptimeSamplerTimer)
+  if (publicDataPrewarmTimer) clearInterval(publicDataPrewarmTimer)
 
   server.close(() => {
     closeDatabase(uptimeDb, 'uptime')