Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/thatdeveloperguy-website / tests/unit/discovery.test.ts |
| Date | Not specified in source |
| Repository | Janady13/thatdeveloperguy-website |
| Revision | 2bea2ebdff0e34494bff597d4fa9c8422a0c3a94 |
| Branch | main |
| Path or archive member | tests/unit/discovery.test.ts |
| Original SHA-256 | 25cf6eb65475684188e5d3d626cc39f7b239b3de367bc8595728cd38d02df023 |
| Git blob SHA | 64ada19fd33960d537a172daef9192e68199b0f5 |
| Reading-copy SHA-256 | 25cf6eb65475684188e5d3d626cc39f7b239b3de367bc8595728cd38d02df023 |
Recovered source transcript
Download complete source text ↓77 lines · 6.3 KB
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { existsSync, readFileSync } from 'node:fs';
import { robotsFor, robotsHeaderFor } from '../../src/engines/seo/crawl/robots-policy.ts';
import { sitemapEntries, sitemapXml, sitemapIndexXml } from '../../src/engines/seo/sitemaps/build-sitemap.ts';
import { llmsDocuments, llmsText } from '../../src/engines/seo/discovery/build-llms.ts';
import type { CompiledPage } from '../../src/contracts/page.ts';
const routes: any[] = [
{ id: 'home', path: '/', template: 'home', publicationStatus: 'published', indexPolicy: 'index', publishedAt: '2026-09-16' },
{ id: 'contact.received', path: '/contact/received', template: 'contact-received', publicationStatus: 'published', indexPolicy: 'noindex' },
{ id: 'capability.cybersecurity', path: '/capabilities/cybersecurity', template: 'capability', publicationStatus: 'demo', indexPolicy: 'index' },
];
test('the demo host disallows everything and sends noindex; production follows the approved matrix and never sends noindex', () => {
assert.equal(robotsFor('demo'), 'User-agent: *\nDisallow: /\n');
assert.equal(robotsHeaderFor('demo'), 'noindex, nofollow');
const production = robotsFor('production');
assert.equal(robotsHeaderFor('production'), 'all');
assert.doesNotMatch(production, /noindex/);
for (const token of ['Googlebot', 'Bingbot', 'GPTBot', 'ClaudeBot', 'Claude-SearchBot', 'PerplexityBot', 'OAI-SearchBot']) assert.match(production, new RegExp(`User-agent: ${token}\\nAllow: /`), `${token} should be allowed per the approved matrix`);
assert.match(production, /Disallow: \/api\//); assert.match(production, /Disallow: \/contact\/received/);
assert.match(production, /Sitemap: https:\/\/thatdeveloperguy\.com\/sitemap\.xml/); assert.match(production, /sitemap-index\.xml/);
});
test('the sitemap holds only published, indexable, canonical pages', () => {
assert.deepEqual(sitemapEntries(routes), ['https://thatdeveloperguy.com/']);
assert.doesNotMatch(sitemapXml(routes), /contact\/received|cybersecurity/);
assert.match(sitemapXml(routes), /<lastmod>2026-09-16<\/lastmod>/);
assert.match(sitemapIndexXml(), /<loc>https:\/\/thatdeveloperguy\.com\/sitemap\.xml<\/loc>/);
});
test('the root llms.txt joins scoped discovery files derived from visible page records', () => {
const pages = [
{ id: 'home', path: '/', heading: 'Lobby', description: 'Choose a service room.', summary: 'Choose the right room.', canonical: 'https://thatdeveloperguy.com/', indexPolicy: 'index', questions: [], related: [] },
{ id: 'government', path: '/government', heading: 'Government Solutions', description: 'Government delivery.', summary: 'Plan and deliver public-sector work.', canonical: 'https://thatdeveloperguy.com/government', indexPolicy: 'index', questions: [{ id: 'q1', question: 'What is offered?', answer: 'Planning and delivery.' }], related: [] },
{ id: 'receipt', path: '/contact/received', heading: 'Receipt', description: 'Private receipt.', summary: 'Private.', canonical: 'https://thatdeveloperguy.com/contact/received', indexPolicy: 'noindex', questions: [], related: [] },
] as unknown as CompiledPage[];
const documents = llmsDocuments(pages, 'demo');
assert.deepEqual(documents.map(document => document.path), ['/llms.txt', '/government/llms.txt']);
assert.equal(llmsText(pages, 'demo'), documents[0]!.content);
assert.match(documents[0]!.content, /non-indexable demonstration release/);
assert.match(documents[0]!.content, /\[Government Solutions\]\(https:\/\/thatdeveloperguy\.com\/government\/llms\.txt\)/);
assert.match(documents[1]!.content, /Site-wide AI discovery index/);
assert.match(documents[1]!.content, /What is offered.*Planning and delivery/);
assert.doesNotMatch(documents.map(document => document.content).join('\n'), /Private receipt|contact\/received/);
});
const truthPolicyPath = process.env.TDG_TRUTH_POLICY ?? '/Users/josephanady/Code/thatdeveloperguy-truth/properties/engine-policy.json';
test('the crawler policy record is approved and internally valid', () => {
const record = JSON.parse(readFileSync(new URL('../../records/crawler-policy.json', import.meta.url), 'utf8'));
assert.equal(record.approval_status, 'approved');
assert.match(record.source, /engine-policy\.json$/);
assert.equal(new Set(record.bots.map((bot: { token: string }) => bot.token)).size, record.bots.length, 'crawler tokens must be unique');
for (const bot of record.bots) assert.ok(['allow', 'deny'].includes(bot.decision), `${bot.token} has an invalid decision`);
assert.deepEqual(record.protected_paths, ['/api/', '/contact/received']);
});
test('the crawler policy record mirrors the external truth node when it is available', { skip: !existsSync(truthPolicyPath) ? 'external truth repository is not present in this checkout' : false }, () => {
const record = JSON.parse(readFileSync(new URL('../../records/crawler-policy.json', import.meta.url), 'utf8'));
const node = JSON.parse(readFileSync(truthPolicyPath, 'utf8')).enginePolicy;
assert.deepEqual(record.bots, node.bots);
});
test('staging directives cannot reach the production host config', async () => {
const { renderHostConfig } = await import('../../src/engines/seo/crawl/host-config.ts');
const demo = renderHostConfig('demo', 'tdg.thatwebhostingguy.com'), production = renderHostConfig('production', 'thatdeveloperguy.com');
assert.doesNotMatch(demo, /auth_basic "/); // open demo by owner decision; TDG_STAGING_GATE=on re-enables the gate
process.env.TDG_STAGING_GATE = 'on'; assert.match(renderHostConfig('demo', 'tdg.thatwebhostingguy.com'), /auth_basic "ThatDeveloperGuy staging"/); delete process.env.TDG_STAGING_GATE; assert.match(demo, /X-Robots-Tag "noindex, nofollow"/);
assert.doesNotMatch(production, /auth_basic "|htpasswd/);
assert.doesNotMatch(production.replace(/location = \/api\/inquiries \{[\s\S]*?\n \}/, ''), /noindex/, 'production may carry noindex only on the API endpoint');
assert.match(production, /live\/thatdeveloperguy\.com\/fullchain/); assert.match(demo, /live\/thatwebhostingguy\.com\/fullchain/);
assert.match(production, /X-Robots-Tag "all"/); assert.match(production, /server_name thatdeveloperguy\.com;/); assert.match(production, /root \/var\/www\/sites\/thatdeveloperguy;/);
assert.doesNotMatch(production, /__[A-Z]+__/, 'no unrendered placeholders');
});