sum more SEO changes

server.cjs

@@ -2449,6 +2449,14 @@ function openGraphType(seo) {
 
 function routeSeo(pathname) {
   const cleanPath = pathname.replace(/\/+$/, '') || '/'
+  const notFoundSeo = {
+    title: "Page Not Found | Toothless' TSS Bot",
+    description: 'The requested Toothless TSS Bot page could not be found.',
+    robots: 'noindex, follow',
+    path: cleanPath === '/' ? '/' : cleanPath,
+    type: 'WebPage',
+    status: 404,
+  }
 
   if (cleanPath.startsWith('/teams/')) {
     const teamName = decodeRouteSegment(cleanPath.slice('/teams/'.length))
@@ -2488,6 +2496,7 @@ function routeSeo(pathname) {
         type: 'BlogPosting',
         publishedAt: post?.date || '',
         author: post?.author || "Toothless' TSS Bot",
+        status: post ? 200 : 404,
       }
     }
   }
@@ -2605,7 +2614,7 @@ function routeSeo(pathname) {
     },
   }
 
-  return byPath[cleanPath] || byPath['/']
+  return byPath[cleanPath] || notFoundSeo
 }
 
 function routeStructuredData(origin, seo, canonicalUrl) {
@@ -2718,9 +2727,18 @@ function htmlWithSeo(req, data) {
     .replaceAll('__TURNSTILE_SESSION__', isTurnstileSessionVerified(req) ? 'verified' : 'required')
 }
 
-function sendHtml(req, res, data, status = 200) {
+function requestPathname(req) {
+  try {
+    return new URL(req.url, pagePublicOrigin(req)).pathname
+  } catch {
+    return '/'
+  }
+}
+
+function sendHtml(req, res, data, status) {
   const html = htmlWithSeo(req, data)
-  send(res, status, html, {
+  const finalStatus = status ?? routeSeo(requestPathname(req)).status ?? 200
+  send(res, finalStatus, html, {
     ...securityHeaders(req, { html: true }),
     'content-type': mimeTypes['.html'],
     'cache-control': 'no-cache',
@@ -3162,12 +3180,12 @@ const server = http.createServer((req, res) => {
     return
   }
 
-  if (req.method === 'GET' && req.url === '/robots.txt') {
+  if ((req.method === 'GET' || req.method === 'HEAD') && req.url === '/robots.txt') {
     sendRobotsTxt(req, res)
     return
   }
 
-  if (req.method === 'GET' && req.url === '/sitemap.xml') {
+  if ((req.method === 'GET' || req.method === 'HEAD') && req.url === '/sitemap.xml') {
     sendSitemapXml(req, res)
     return
   }