Fix docs indexing with sitemap, canonical URLs, and root redirects

app/lib/shared.ts

@@ -1,4 +1,6 @@
 export const appName = 'Talo'
 export const docsRoute = '/'
 
+export const siteOrigin = 'https://docs.trytalo.com'
+
 export const baseApiUrl = import.meta.env.DEV ? 'http://localhost:3000' : 'https://api.trytalo.com'

app/routes.ts

@@ -2,7 +2,7 @@ import { index, route, type RouteConfig } from '@react-router/dev/routes'
 
 export default [
   index('routes/home.tsx'),
-  route('intro', 'routes/intro-redirect.tsx'),
   route('api/search', 'routes/search.ts'),
+  route('sitemap.xml', 'routes/sitemap.ts'),
   route(':version/*', 'routes/docs.tsx'),
 ] satisfies RouteConfig

app/routes/docs.tsx

@@ -8,6 +8,7 @@ import { Feedback } from '@/components/feedback/client'
 import { useMDXComponents } from '@/components/mdx'
 import { onPageFeedback } from '@/lib/feedback'
 import { baseOptions } from '@/lib/layout.shared'
+import { siteOrigin } from '@/lib/shared'
 import { docs, source } from '@/lib/source'
 import type { Route } from './+types/docs'
 
@@ -23,11 +24,12 @@ export async function loader({ params }: Route.LoaderArgs) {
 
   return {
     path: page.path,
+    url: page.url,
     pageTree: await source.serializePageTree(source.getPageTree()),
   }
 }
 
-function Content({ path }: { path: string }) {
+function Content({ path, url }: { path: string; url: string }) {
   const page = docs.getPage(path)
   if (!page) {
     throw new Error(`unknown page: ${path}`)
@@ -48,6 +50,7 @@ function Content({ path }: { path: string }) {
       <title>
         {path === 'index.mdx' ? 'Talo - open source, self-hostable game backend' : page.title}
       </title>
+      <link rel='canonical' href={`${siteOrigin}${url}`} />
       <meta name='description' content={page.description} />
       <DocsTitle>{page.title}</DocsTitle>
       <DocsDescription>{page.description}</DocsDescription>
@@ -62,11 +65,11 @@ function Content({ path }: { path: string }) {
 }
 
 export default function Page({ loaderData }: Route.ComponentProps) {
-  const { path, pageTree } = useFumadocsLoader(loaderData)
+  const { path, url, pageTree } = useFumadocsLoader(loaderData)
 
   return (
     <DocsLayout {...baseOptions()} tree={pageTree}>
-      <Content path={path} />
+      <Content path={path} url={url} />
     </DocsLayout>
   )
 }

app/routes/sitemap.ts

@@ -0,0 +1,14 @@
+import { siteOrigin } from '@/lib/shared'
+import { source } from '@/lib/source'
+
+export function loader() {
+  const urls = source.getPages().map((page) => `<loc>${siteOrigin}${page.url}</loc>`)
+  const body = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls.join('\n')}
+</urlset>`
+
+  return new Response(body, {
+    headers: { 'content-type': 'application/xml; charset=utf-8' },
+  })
+}

public/robots.txt

@@ -1,3 +1,4 @@
 User-agent: *
 Disallow:
 # Algolia-Crawler-Verif: 50BA364995958603
+Sitemap: https://docs.trytalo.com/sitemap.xml

react-router.config.ts

@@ -10,7 +10,6 @@ export default {
     const paths: string[] = []
 
     for (const path of getStaticPaths()) {
-      // `/` navigates to the default version via the client component
       if (path !== '/') {
         paths.push(path)
       }
@@ -21,6 +20,8 @@ export default {
       paths.push(getUrl(slugs))
     }
 
+    paths.push('/sitemap.xml')
+
     return paths
   },
 } satisfies Config

workers/app.ts

@@ -6,6 +6,10 @@ type Env = {}
 // Pre-versioning paths (e.g. /docs/godot/install) redirect to the default version.
 const DEFAULT_VERSION = '1.x'
 const LEGACY_SECTIONS = ['godot', 'unity', 'http', 'sockets', 'selfhosting', 'integrations']
+const INTRO_SEGMENT = 'intro'
+// React Router sets this while prerendering the SPA fallback during `pnpm build`.
+// That request must render the shell, so it must skip the `/` redirect below.
+const SPA_FALLBACK_HEADER = 'X-React-Router-SPA-Mode'
 
 const requestHandler = createRequestHandler(
   () => import('virtual:react-router/server-build'),
@@ -15,6 +19,14 @@ const requestHandler = createRequestHandler(
 export default {
   async fetch(request) {
     const { pathname, origin } = new URL(request.url)
+    // `/` carries no version, so send it to the default one.
+    if (pathname === '/' && !request.headers.has(SPA_FALLBACK_HEADER)) {
+      return Response.redirect(`${origin}/${DEFAULT_VERSION}`, 301)
+    }
+    // `/intro` pre-dates the versioned paths and now lives at `/1.x`.
+    if (pathname === '/intro' || pathname === '/intro/') {
+      return Response.redirect(`${origin}/${DEFAULT_VERSION}`, 301)
+    }
     // Old `/docs/*` URLs redirect to the prefix-less paths.
     if (pathname === '/docs' || pathname === '/docs/') {
       return Response.redirect(`${origin}/${DEFAULT_VERSION}`, 301)
@@ -23,6 +35,9 @@ export default {
     if (match) {
       const rest = match[1]
       const [first] = rest.split('/')
+      if (first === INTRO_SEGMENT) {
+        return Response.redirect(`${origin}/${DEFAULT_VERSION}`, 301)
+      }
       const target = LEGACY_SECTIONS.includes(first) ? `/${DEFAULT_VERSION}/${rest}` : `/${rest}`
       return Response.redirect(`${origin}${target}`, 301)
     }

wrangler.jsonc

@@ -5,5 +5,9 @@
   "observability": {
     "enabled": true,
   },
+  "assets": {
+    "html_handling": "drop-trailing-slash",
+    "run_worker_first": ["/"],
+  },
   "main": "./workers/app.ts",
 }