Disable robots for preview pages

Author: botder

astro.config.mjs

@@ -4,7 +4,7 @@ import starlight from '@astrojs/starlight';
 import mtasaStarlightThemePlugin from '@multitheftauto/starlight-theme-mtasa';
 import { SITE_TITLE, SITE_URL, SITE_PREVIEW_URL } from './src/content.constants';
 
-const isDeploy = process.env.CI === 'true';
+const isDeploy = process.env.CI === 'true' && process.env.CI_PREVIEW !== 'true';
 const siteBaseUrl = isDeploy ? SITE_URL : SITE_PREVIEW_URL;
 
 export default defineConfig({

src/pages/robots.txt.ts

@@ -1,11 +1,13 @@
 import type { APIRoute } from 'astro';
 
-const getRobotsTxt = (sitemapURL: URL) => `
-User-agent: *
-Allow: /
+const getRobotsTxt = (sitemapURL: URL) => {
+    let previewToggle = sitemapURL.href.includes('.preview.') ? 'Disallow' : 'Allow';
+    return `User-agent: *
+${previewToggle}: /
 
 Sitemap: ${sitemapURL.href}
 `;
+}
 
 export const GET: APIRoute = ({ site }) => {
     const sitemapURL = new URL('sitemap-index.xml', site);