ai generated solutions to our ai generated problems

2026-06-20 00:05:10 +01:00
parent 736fe40e75
commit a60999a54e
3 changed files with 254 additions and 7 deletions
@@ -48,6 +48,12 @@ const ANALYTICS_DATABASE_FILE = process.env.ANALYTICS_DATABASE_FILE || 'viewers.
 const ANALYTICS_RETENTION_DAYS = Number(process.env.ANALYTICS_RETENTION_DAYS || 30)
 const ANALYTICS_ACTIVE_WINDOW_SECONDS = Number(process.env.ANALYTICS_ACTIVE_WINDOW_SECONDS || 75)
 const API_CACHE_TTL_MS = Number(process.env.API_CACHE_TTL_MS || 15000)
+const PUBLIC_DATA_CACHE_DIR = path.resolve(
+  expandHome(process.env.PUBLIC_DATA_CACHE_DIR || path.join(UPTIME_STORAGE_DIR, 'public-data')),
+)
+const PUBLIC_DATA_CACHE_FRESH_MS = Number(process.env.PUBLIC_DATA_CACHE_FRESH_MS || 60 * 1000)
+const PUBLIC_DATA_CACHE_STALE_MS = Number(process.env.PUBLIC_DATA_CACHE_STALE_MS || 24 * 60 * 60 * 1000)
+const PUBLIC_DATA_PREWARM_INTERVAL_MS = Number(process.env.PUBLIC_DATA_PREWARM_INTERVAL_MS || PUBLIC_DATA_CACHE_FRESH_MS)
 const API_RATE_LIMIT_WINDOW_MS = Number(process.env.API_RATE_LIMIT_WINDOW_MS || 60000)
 const API_RATE_LIMIT_MAX = Number(process.env.API_RATE_LIMIT_MAX || 120)
 const TURNSTILE_SECRET_KEY = process.env.TURNSTILE_SECRET_KEY || ''
@@ -197,15 +203,208 @@ const jsonHeaders = {
 }

 const apiCache = new Map()
+const publicDataRefreshes = new Set()
 const rateLimits = new Map()
 let uptimeDb = null
 let analyticsDb = null
 let latestUptimeSnapshot = null
+let publicDataPrewarmTimer = null

 function sendJson(res, status, body, headers = {}) {
  send(res, status, JSON.stringify(body), { ...jsonHeaders, ...headers })
 }

+function publicDataKey(value) {
+  return encodeURIComponent(String(value || '').trim()).replace(/%/g, '~')
+}
+
+function publicDataCachePathForUrl(requestUrl) {
+  const pathname = requestUrl.pathname
+  const params = requestUrl.searchParams
+
+  if (pathname === '/api/tss/leaderboard/teams') {
+    const limit = Number(params.get('limit') || 100)
+    if (limit === 4) return path.join(PUBLIC_DATA_CACHE_DIR, 'home-teams.json')
+    if (limit === 100) return path.join(PUBLIC_DATA_CACHE_DIR, 'leaderboard-teams.json')
+    return null
+  }
+
+  if (pathname === '/api/tss/leaderboard/players') {
+    const limit = Number(params.get('limit') || 100)
+    return limit === 100 ? path.join(PUBLIC_DATA_CACHE_DIR, 'leaderboard-players.json') : null
+  }
+
+  if (pathname === '/api/tss/games/recent') {
+    const limit = Number(params.get('limit') || 50)
+    return limit === 50 ? path.join(PUBLIC_DATA_CACHE_DIR, 'recent-games.json') : null
+  }
+
+  let match = pathname.match(/^\/api\/tss\/teams\/([^/]+)$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'teams', `${publicDataKey(decodeURIComponent(match[1]))}.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/teams\/([^/]+)\/games$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'teams', `${publicDataKey(decodeURIComponent(match[1]))}.games.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/player\/([0-9]{1,32})$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'players', `${publicDataKey(match[1])}.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/games\/([A-Za-z0-9_-]{1,96})$/)
+  if (match && (params.get('lang') || 'en') === 'en' && [...params.keys()].every((key) => key === 'lang')) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'games', `${publicDataKey(match[1])}.json`)
+  }
+
+  match = pathname.match(/^\/api\/tss\/games\/([A-Za-z0-9_-]{1,96})\/logs$/)
+  if (match && ![...params.keys()].length) {
+    return path.join(PUBLIC_DATA_CACHE_DIR, 'games', `${publicDataKey(match[1])}.logs.json`)
+  }
+
+  return null
+}
+
+function safePublicDataPath(requestPath) {
+  let decoded = '/'
+  try {
+    decoded = decodeURIComponent(requestPath)
+  } catch {
+    return null
+  }
+
+  const relative = decoded.replace(/^\/data\/?/, '')
+  if (!relative || relative.includes('\0')) return null
+
+  const filePath = path.resolve(PUBLIC_DATA_CACHE_DIR, relative)
+  const relativeToCache = path.relative(PUBLIC_DATA_CACHE_DIR, filePath)
+  if (relativeToCache.startsWith('..') || path.isAbsolute(relativeToCache)) return null
+  return filePath
+}
+
+function sendPublicDataFile(req, res, filePath, status = 200, extraHeaders = {}) {
+  fs.readFile(filePath, (error, data) => {
+    if (error) {
+      sendJson(res, 404, { error: 'Snapshot not found' })
+      return
+    }
+
+    send(res, status, data, {
+      'content-type': 'application/json; charset=utf-8',
+      'cache-control': 'public, max-age=30, stale-while-revalidate=300',
+      ...extraHeaders,
+    })
+  })
+}
+
+function cachedPublicData(filePath) {
+  if (!filePath) return null
+
+  try {
+    const stat = fs.statSync(filePath)
+    if (!stat.isFile()) return null
+    const age = Date.now() - stat.mtimeMs
+    if (age > PUBLIC_DATA_CACHE_STALE_MS) return null
+    return { filePath, age, fresh: age <= PUBLIC_DATA_CACHE_FRESH_MS }
+  } catch {
+    return null
+  }
+}
+
+function writePublicDataFile(filePath, body) {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true })
+  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`
+  fs.writeFileSync(tmpPath, body)
+  fs.renameSync(tmpPath, filePath)
+}
+
+function refreshPublicData(filePath, target) {
+  if (!filePath || publicDataRefreshes.has(filePath)) return
+  publicDataRefreshes.add(filePath)
+
+  fetchUpstreamJsonBuffer(target)
+    .then((body) => writePublicDataFile(filePath, body))
+    .catch((error) => {
+      console.warn(`public data refresh failed for ${target.pathname}: ${error.message}`)
+    })
+    .finally(() => publicDataRefreshes.delete(filePath))
+}
+
+function publicDataPrewarmTargets() {
+  return [
+    '/api/tss/leaderboard/teams?limit=100',
+    '/api/tss/leaderboard/players?limit=100',
+    '/api/tss/leaderboard/teams?limit=4',
+    '/api/tss/games/recent?limit=50',
+  ].map((requestPath) => {
+    const requestUrl = new URL(requestPath, 'http://localhost')
+    return {
+      requestUrl,
+      target: new URL(requestPath, API_UPSTREAM),
+      filePath: publicDataCachePathForUrl(requestUrl),
+    }
+  }).filter((entry) => entry.filePath)
+}
+
+function prewarmPublicDataCache() {
+  for (const { filePath, target } of publicDataPrewarmTargets()) {
+    const current = cachedPublicData(filePath)
+    if (current?.fresh) continue
+    refreshPublicData(filePath, target)
+  }
+}
+
+function startPublicDataPrewarmer() {
+  fs.mkdirSync(PUBLIC_DATA_CACHE_DIR, { recursive: true })
+  prewarmPublicDataCache()
+  publicDataPrewarmTimer = setInterval(prewarmPublicDataCache, PUBLIC_DATA_PREWARM_INTERVAL_MS)
+  publicDataPrewarmTimer.unref?.()
+}
+
+function fetchUpstreamJsonBuffer(target) {
+  return new Promise((resolve, reject) => {
+    const client = target.protocol === 'https:' ? https : http
+    const chunks = []
+    let bytes = 0
+    const proxy = client.request(
+      target,
+      {
+        method: 'GET',
+        headers: {
+          accept: 'application/json',
+          host: target.host,
+          'user-agent': 'tssbot-web-cache',
+        },
+      },
+      (proxyRes) => {
+        const status = proxyRes.statusCode || 502
+        const contentType = String(proxyRes.headers['content-type'] || '')
+        if (status < 200 || status >= 300 || !contentType.includes('application/json')) {
+          proxyRes.resume()
+          reject(new Error(`Upstream returned ${status}`))
+          return
+        }
+
+        proxyRes.on('data', (chunk) => {
+          bytes += chunk.length
+          if (bytes > MAX_UPSTREAM_BODY_BYTES) {
+            proxy.destroy(new Error('Upstream response too large'))
+            return
+          }
+          chunks.push(chunk)
+        })
+        proxyRes.on('end', () => resolve(Buffer.concat(chunks)))
+      },
+    )
+
+    proxy.setTimeout(SERVER_REQUEST_TIMEOUT_MS, () => proxy.destroy(new Error('Upstream request timed out')))
+    proxy.on('error', reject)
+    proxy.end()
+  })
+}
+
 function requestJson(url, timeoutMs = 10000) {
  return new Promise((resolve, reject) => {
    const target = new URL(url)
@@ -1713,6 +1912,17 @@ function proxyRequest(req, res) {
    return sendJson(res, 404, { error: 'API route not found' })
  }

+  const requestUrl = new URL(req.url, `http://${req.headers.host || 'localhost'}`)
+  const publicDataFile = req.method === 'GET' ? publicDataCachePathForUrl(requestUrl) : null
+  const publicData = cachedPublicData(publicDataFile)
+  if (publicData?.fresh) {
+    return sendPublicDataFile(req, res, publicData.filePath, 200, { 'x-tssbot-cache': 'public-data-hit' })
+  }
+  if (publicData) {
+    refreshPublicData(publicData.filePath, target)
+    return sendPublicDataFile(req, res, publicData.filePath, 200, { 'x-tssbot-cache': 'public-data-stale' })
+  }
+
  const cacheKey = req.method === 'GET' ? target.toString() : ''
  const cached = cacheKey ? apiCache.get(cacheKey) : null
  if (cached && cached.expiresAt > Date.now()) {
@@ -1732,10 +1942,13 @@ function proxyRequest(req, res) {
      },
    },
    (proxyRes) => {
+      const statusCode = proxyRes.statusCode || 502
+      const contentType = String(proxyRes.headers['content-type'] || '')
+      const shouldCacheJson = statusCode >= 200 && statusCode < 300 && contentType.includes('application/json')
      const headers = {
        ...proxyRes.headers,
        ...securityHeaders(req),
-        'cache-control': 'private, max-age=15',
+        'cache-control': publicDataFile ? 'public, max-age=30, stale-while-revalidate=300' : 'private, max-age=15',
      }

      delete headers['access-control-allow-origin']
@@ -1744,7 +1957,7 @@ function proxyRequest(req, res) {
      delete headers['access-control-allow-headers']
      delete headers['access-control-expose-headers']

-      res.writeHead(proxyRes.statusCode || 502, headers)
+      res.writeHead(statusCode, headers)

      proxyRes.on('data', (chunk) => {
        proxiedBytes += chunk.length
@@ -1753,18 +1966,26 @@ function proxyRequest(req, res) {
          res.destroy()
          return
        }
-        if (cacheKey && (proxyRes.statusCode || 0) >= 200 && (proxyRes.statusCode || 0) < 300) {
+        if (cacheKey && shouldCacheJson) {
          responseChunks.push(chunk)
        }
      })

      proxyRes.on('end', () => {
        if (cacheKey && responseChunks.length) {
+          const body = Buffer.concat(responseChunks)
          apiCache.set(cacheKey, {
-            body: Buffer.concat(responseChunks),
+            body,
            headers,
            expiresAt: Date.now() + API_CACHE_TTL_MS,
          })
+          if (publicDataFile) {
+            try {
+              writePublicDataFile(publicDataFile, body)
+            } catch (error) {
+              console.warn(`could not write public data cache for ${requestUrl.pathname}: ${error.message}`)
+            }
+          }
        }
      })

@@ -2459,6 +2680,16 @@ const server = http.createServer((req, res) => {
      serveReplayMinimap(req, res)
      return
    }
+
+    if (pathname.startsWith('/data/')) {
+      const filePath = safePublicDataPath(pathname)
+      if (!filePath) {
+        sendJson(res, 400, { error: 'Bad request' })
+        return
+      }
+      sendPublicDataFile(req, res, filePath)
+      return
+    }
  }

  if (req.method === 'GET' && req.url.startsWith('/vehicle-icons/')) {
@@ -2487,10 +2718,14 @@ server.listen(PORT, '0.0.0.0', () => {
  }
  console.log(`storing uptime snapshots in ${path.join(uptimeStoragePath(), UPTIME_DATABASE_FILE)}`)
  console.log(`storing viewer analytics in ${path.join(uptimeStoragePath(), ANALYTICS_DATABASE_FILE)}`)
+  console.log(`storing public data cache in ${PUBLIC_DATA_CACHE_DIR}`)
  if (!TURNSTILE_SECRET_KEY) {
    console.warn('TURNSTILE_SECRET_KEY is not set — Turnstile verification is disabled and gated endpoints will accept any request')
  }
-  if (RUN_BACKGROUND_JOBS) startUptimeSampler()
+  if (RUN_BACKGROUND_JOBS) {
+    startUptimeSampler()
+    startPublicDataPrewarmer()
+  }
  process.send?.('ready')
 })

@@ -2511,6 +2746,7 @@ function shutdown() {
  shuttingDown = true

  if (uptimeSamplerTimer) clearInterval(uptimeSamplerTimer)
+  if (publicDataPrewarmTimer) clearInterval(publicDataPrewarmTimer)

  server.close(() => {
    closeDatabase(uptimeDb, 'uptime')