Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/thatdeveloperguy-website / src/engines/seo/crawl/robots-policy.ts |
| Date | Not specified in source |
| Repository | Janady13/thatdeveloperguy-website |
| Revision | 2bea2ebdff0e34494bff597d4fa9c8422a0c3a94 |
| Branch | main |
| Path or archive member | src/engines/seo/crawl/robots-policy.ts |
| Original SHA-256 | f673107797036c6227d1419edb5938e6942e63c268d6b781f2b0904d77837818 |
| Git blob SHA | 1f4201e52a0afaa0ec16ae8d11bf37709d2064fd |
| Reading-copy SHA-256 | f673107797036c6227d1419edb5938e6942e63c268d6b781f2b0904d77837818 |
Recovered source transcript
Download complete source text ↓16 lines · 1.1 KB
import policy from '../../../../records/crawler-policy.json' with { type: 'json' };
import { PRODUCTION_ORIGIN } from '../urls/production-origin.ts';
export type ReleaseKind = 'demo' | 'production';
/** robots.txt has exactly one generated owner. Demo hosts disallow everything; production follows the owner's approved bot matrix. */
export function robotsFor(kind: ReleaseKind): string {
if (kind === 'demo') return 'User-agent: *\nDisallow: /\n';
const groups = policy.bots.map(bot => `User-agent: ${bot.token}\n${bot.decision === 'allow' ? 'Allow: /' : 'Disallow: /'}\n`).join('\n');
const protectedPaths = policy.protected_paths.map(p => `Disallow: ${p}`).join('\n');
return `# Approved by ${policy.approved_by}, ${policy.approved_at}\n${groups}\nUser-agent: *\nAllow: /\n${protectedPaths}\n\nSitemap: ${PRODUCTION_ORIGIN}/sitemap.xml\nSitemap: ${PRODUCTION_ORIGIN}/sitemap-index.xml\n`;
}
/** X-Robots-Tag for the host config: staging is noindex regardless of what robots.txt says. */
export function robotsHeaderFor(kind: ReleaseKind): string { return kind === 'demo' ? 'noindex, nofollow' : 'all'; }