A continuation of Janady13/crawler-signal-reference / framework-http-seo-headers.md. The complete source identity, every recorded occurrence, and linked equations are on its first page.
Source transcript · continued
Download complete source text ↓2,062 lines · 82.2 KB
server_name www.example.com;
ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
return 301 https://example.com$request_uri;
}
# Main canonical server
server {
listen 443 ssl;
listen [::]:443 ssl;
http2 on;
listen 443 quic reuseport;
listen [::]:443 quic reuseport;
server_name example.com;
ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem;
root /var/www/sites/example.com;
index index.html;
# HTTP/3 advertisement
add_header Alt-Svc 'h3=":443"; ma=86400' always;
# ===== HTML PAGES =====
location ~* \.html$ {
include snippets/common-security-headers.conf;
# Baseline indexing directives
add_header X-Robots-Tag "max-snippet:200, max-image-preview:large, max-video-preview:-1" always;
# Preload critical resources
add_header Link '</css/critical.css>; rel="preload"; as="style", </fonts/manrope.woff2>; rel="preload"; as="font"; crossorigin, </images/hero.webp>; rel="preload"; as="image"; fetchpriority="high"' always;
# Caching from framework-http-caching-headers.md
add_header Cache-Control "public, max-age=0, must-revalidate" always;
# Language from framework-http-content-headers.md
add_header Content-Language "en-US" always;
}
# ===== PDFs (noindex, canonical to HTML landing) =====
location ~ ^/whitepapers/(.*)\.pdf$ {
include snippets/common-security-headers.conf;
add_header X-Robots-Tag "noindex" always;
add_header Link '<https://example.com/whitepapers/$1>; rel="canonical"' always;
add_header Content-Disposition "attachment" always;
add_header Cache-Control "public, max-age=86400" always;
}
# ===== IMAGES (allow image search indexing) =====
location ~* \.(jpg|jpeg|png|webp|avif|svg|gif)$ {
include snippets/common-security-headers.conf;
add_header Cache-Control "public, max-age=2592000" always;
# No X-Robots-Tag: default behavior allows image indexing
}
# ===== FONTS, CSS, JS (no indexing needed; cache forever) =====
location ~* \.(woff2|woff|ttf|otf|css|js|mjs)$ {
include snippets/common-security-headers.conf;
add_header Cache-Control "public, max-age=31536000, immutable" always;
}
# ===== INTERNAL SEARCH (noindex but follow) =====
location /search {
include snippets/common-security-headers.conf;
add_header X-Robots-Tag "noindex, follow" always;
}
# ===== API (sidecar, noindex) =====
location /api/ {
include snippets/common-security-headers.conf;
add_header X-Robots-Tag "noindex, nofollow" always;
# Early Hints for dynamic responses
early_hints 103;
add_header Link '</css/dashboard.css>; rel="preload"; as="style"' always;
proxy_pass http://127.0.0.1:9090;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# ===== ROBOTS.TXT, SITEMAPS, AI/LLM FILES =====
location = /robots.txt {
expires 1h;
add_header Cache-Control "public, max-age=3600" always;
}
location = /sitemap.xml {
expires 10m;
add_header Cache-Control "public, max-age=600" always;
}
location ~* /(llms\.txt|ai\.txt|humans\.txt|\.well-known/security\.txt)$ {
expires 1d;
add_header Cache-Control "public, max-age=86400" always;
}
# ===== LEGACY URL MAP REDIRECTS =====
if ($legacy_redirect != "") {
return 301 $legacy_redirect;
}
# ===== DEFAULT =====
location / {
include snippets/common-security-headers.conf;
add_header X-Robots-Tag "max-snippet:200, max-image-preview:large, max-video-preview:-1" always;
add_header Content-Language "en-US" always;
add_header Cache-Control "public, max-age=0, must-revalidate" always;
try_files $uri $uri/ $uri.html =404;
}
}
# Legacy URL map (defined at http level in nginx.conf or included)
# map $request_uri $legacy_redirect {
# default "";
# /old/page-1.html /new/page-1;
# /old/page-2.html /new/page-2;
# }
```
The shared security header snippet at `/etc/nginx/snippets/common-security-headers.conf`:
```nginx
add_header X-Frame-Options "SAMEORIGIN" always;
add_header X-Content-Type-Options "nosniff" always;
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;
add_header Permissions-Policy "camera=(), microphone=(), geolocation=(), payment=()" always;
```
After deploying:
```bash
nginx -t && systemctl reload nginx
```
---
## 11. AUDIT CHECKLIST
Run through these 50 items for any production site.
### X-Robots-Tag
1. [ ] Production HTML pages have `X-Robots-Tag: max-snippet:200, max-image-preview:large, max-video-preview:-1` (or appropriate values, NOT `noindex`).
2. [ ] Staging or development subdomains have `X-Robots-Tag: noindex, nofollow`.
3. [ ] PDFs at customer facing URLs have `X-Robots-Tag: noindex` (unless they should be indexed).
4. [ ] CSV exports and other downloadable data files have `X-Robots-Tag: noindex`.
5. [ ] API endpoints have `X-Robots-Tag: noindex, nofollow`.
6. [ ] Internal search result pages have `X-Robots-Tag: noindex, follow`.
7. [ ] No production HTML page has `X-Robots-Tag: noindex` accidentally.
8. [ ] No directive uses deprecated tokens (`noodp`, `noydir`, `noyaca`).
9. [ ] Bot prefixed directives (`googlebot:`, `bingbot:`) use valid bot tokens.
10. [ ] `unavailable_after` dates use RFC 850 / RFC 822 format.
11. [ ] `nosnippet` is used only when intentional (it blocks AI Overviews too).
12. [ ] X-Robots-Tag from HTTP and `<meta name="robots">` from HTML do not conflict on the same page.
### Link header
13. [ ] HTML pages preload critical CSS, fonts, and hero image via `Link` HTTP header.
14. [ ] Font preloads include `crossorigin` parameter.
15. [ ] All preload URLs use absolute or relative paths that match the actual usage in HTML.
16. [ ] Preload `as` values are correct (`style`, `script`, `font`, `image`, etc).
17. [ ] LCP candidate image has `fetchpriority="high"`.
18. [ ] PDFs and other non HTML assets that need canonicalization use `Link: <...>; rel="canonical"`.
19. [ ] Hreflang annotations form a reciprocal cluster (every variant lists every other variant).
20. [ ] Hreflang cluster includes `x-default`.
21. [ ] No `rel=prev` or `rel=next` headers (Google ignored these in 2019).
22. [ ] Early Hints (103) is enabled for dynamic responses where TTFB exceeds 200 ms.
23. [ ] Nginx version is 1.25+ if Early Hints is used.
### Location and redirects
24. [ ] HTTP requests to port 80 always redirect to HTTPS with 301.
25. [ ] www to non www (or vice versa) is consistent and uses 301.
26. [ ] No redirect chain exceeds 1 hop.
27. [ ] All 301 redirect targets return 200 OK (not another redirect or error).
28. [ ] Permanent redirects use 301 or 308 (not 302 or 307).
29. [ ] Temporary redirects use 302 or 307.
30. [ ] API version migration redirects use 308 to preserve POST method.
31. [ ] No redirect endpoint accepts unvalidated URL parameters from user input (no open redirects).
32. [ ] Trailing slash policy is consistent site wide.
33. [ ] Internal links point directly to the canonical URL (no internal links to redirect sources).
34. [ ] Redirect targets are absolute URLs or path absolute (no protocol relative or relative redirects).
35. [ ] SSL certificate covers all redirect target hostnames.
### Cross cutting
36. [ ] `nginx -t` passes without warnings.
37. [ ] `nginx -T` shows all expected SEO directives in effect for each server block.
38. [ ] Search Console URL Inspection shows correct indexing status for sample URLs.
39. [ ] No URL is in Google's index that should be noindex.
40. [ ] No URL is noindex that should be in Google's index.
41. [ ] No URL is blocked by robots.txt that needs to be noindex (the noindex would not be seen).
42. [ ] PDF count in Search Console matches expected count (audits whether unwanted PDFs are indexed).
43. [ ] Hreflang errors in Search Console are at zero.
44. [ ] Coverage report in Search Console shows no unexpected "Excluded by noindex" entries.
45. [ ] Redirect chain audit (run periodically) shows zero chains over 1 hop.
46. [ ] All preload directives have corresponding usage in HTML (no orphan preloads wasting bandwidth).
47. [ ] LCP image is preloaded with fetchpriority=high.
48. [ ] Early Hints (if used) actually arrives before final response (verified via `curl --http2`).
49. [ ] Open redirect scanner reports clean.
50. [ ] Internal link audit shows no broken links to redirected URLs.
A site that passes all 50 has indexing, resource hints, and forwarding correctly configured.
---
## 12. COMMON PITFALLS
Fifteen patterns to recognize and avoid.
**Pitfall 1: noindex behind robots.txt block.**
Symptom: page stays indexed forever.
Why it breaks: robots.txt prevents fetching; crawler never sees the noindex directive.
Fix: remove robots.txt block, let crawler fetch and process noindex, then optionally re add.
**Pitfall 2: Conflicting X-Robots-Tag and meta robots.**
Symptom: page sometimes indexed, sometimes not; behavior inconsistent.
Why it breaks: both signals are read; the most restrictive wins, but team members assume one or the other is authoritative.
Fix: align both. Pick one as the source of truth (typically HTTP header for non HTML, HTML tag for HTML pages) and maintain consistency.
**Pitfall 3: noindex on a redirected URL.**
Symptom: the noindex header is wasted; the 301 itself handles deindexing.
Why it breaks: a 301 response signals the URL has moved; crawlers process the redirect, not the directives on the old URL.
Fix: remove noindex from redirect responses unless you have a specific reason.
**Pitfall 4: Canonical pointing to a redirected URL.**
Symptom: index signals are diluted; Google follows the chain and selects its own canonical.
Why it breaks: canonical should point to the final canonical URL, not to an intermediate that redirects.
Fix: audit all canonicals. Each must resolve to a 200 OK page that self references as canonical.
**Pitfall 5: Preload with wrong `as` value.**
Symptom: browser fetches the resource twice (preload, then real load).
Why it breaks: the browser cannot match the preloaded resource to the actual request because the priority and headers differ.
Fix: ensure `as` matches the resource type exactly. Font preload requires `as=font` AND `crossorigin`. Image is `as=image`. Script is `as=script`. Style is `as=style`.
**Pitfall 6: Preload of a resource the page does not use.**
Symptom: bandwidth wasted; console warning "preloaded but not used".
Why it breaks: a developer added a preload speculatively, then the corresponding HTML reference was removed or changed.
Fix: audit preloads. Remove any not actually used within a few seconds of page load.
**Pitfall 7: Hreflang cluster missing reciprocity.**
Symptom: Search Console hreflang errors; Google falls back to default behavior.
Why it breaks: hreflang requires every variant to list every other variant including itself.
Fix: build hreflang generation programmatically so all variants always list the full set.
**Pitfall 8: Redirect chain longer than 1 hop.**
Symptom: crawl budget waste; PageRank consolidation slow.
Why it breaks: nobody updated the source of a redirect when the destination itself was redirected.
Fix: audit periodically with curl `-sLI`. Collapse to direct redirects.
**Pitfall 9: Open redirect endpoint.**
Symptom: security alert; phishing campaigns abuse the redirector.
Why it breaks: endpoint accepts arbitrary target URL from user input without validation.
Fix: allow list, signature, or only accept path components. See Section 7.4.
**Pitfall 10: Wrong redirect code for HTTP method preservation.**
Symptom: POST becomes GET after redirect; API calls fail.
Why it breaks: 301, 302, 303 historically change POST to GET.
Fix: use 307 (temporary) or 308 (permanent) for redirects that must preserve POST.
**Pitfall 11: 301 cached forever, hard to undo.**
Symptom: changed the redirect target but browsers still go to the old destination.
Why it breaks: per HTTP spec, browsers may cache 301 indefinitely.
Fix prevention: use 302 for redirects you might change in the future. Fix existing: change the source URL or wait for cache expiration.
**Pitfall 12: Self redirecting URL.**
Symptom: redirect loop; ERR_TOO_MANY_REDIRECTS.
Why it breaks: a location block redirects, and the target also matches the same location pattern.
Fix: tighten the regex match, or check the request URI before redirecting.
**Pitfall 13: Redirect target on a different SSL certificate.**
Symptom: browser shows certificate warning after redirect.
Why it breaks: redirecting `www.example.com` to `example.com` requires the certificate to cover `example.com`.
Fix: ensure the cert covers all redirect targets. With Let's Encrypt, include all hostnames in the `-d` flags.
**Pitfall 14: Add_header inheritance trap on X-Robots-Tag.**
Symptom: noindex applied to staging site disappears from specific file types.
Why it breaks: a location block with `add_header` wipes parent declarations.
Fix: snippet include pattern in every location.
**Pitfall 15: Early Hints emitted but contains preload for a non existent resource.**
Symptom: 404 in network panel for the preloaded resource; LCP not improved.
Why it breaks: preload URL is wrong or the resource was renamed.
Fix: audit Early Hints preload URLs. Each must resolve to 200 OK.
---
## 13. DIAGNOSTIC COMMANDS
Reference of every command useful for SEO header investigation.
### Inspect SEO headers
```bash
# Full SEO family at once
curl -sI https://example.com/page.html | grep -iE "^(x-robots-tag|link|location|content-language):"
# As Googlebot
curl -sI -A "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" \
https://example.com/page.html
# As Bingbot
curl -sI -A "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)" \
https://example.com/page.html
# As ClaudeBot
curl -sI -A "Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)" \
https://example.com/page.html
# As GPTBot
curl -sI -A "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.0; +https://openai.com/gptbot" \
https://example.com/page.html
```
### X-Robots-Tag verification
```bash
# Confirm directive
curl -sI https://example.com/page.html | grep -i x-robots-tag
# Check all non HTML file types
for ext in pdf png jpg svg woff2 json xml csv zip; do
URL="https://example.com/test.$ext"
echo "=== $URL ==="
curl -sI "$URL" 2>/dev/null | grep -i x-robots-tag
done
# Check for accidental noindex on production pages
for url in / /about /services /contact /blog; do
echo "=== $url ==="
curl -sI "https://example.com$url" | grep -i x-robots-tag
done
# Compare HTTP and HTML signals
echo "HTTP:"; curl -sI https://example.com/page.html | grep -i x-robots-tag
echo "HTML:"; curl -s https://example.com/page.html | grep -oE '<meta[^>]*name="robots"[^>]*>'
```
### Link header verification
```bash
# Confirm Link header presence and parse
curl -sI https://example.com/ | grep -i '^link:' | sed 's/, /\n /g'
# Check canonical
curl -sI https://example.com/page.html | grep -oE 'rel="canonical"[^,]*'
# Check hreflang cluster
curl -sI https://example.com/ | grep -oE 'hreflang="[^"]*"' | sort -u
# Verify preload targets resolve
for url in $(curl -sI https://example.com/ | grep -i '^link:' | grep -oE '<[^>]+>' | tr -d '<>'); do
STATUS=$(curl -sI "$url" | head -1)
echo "$url -> $STATUS"
done
# Test Early Hints
curl -sv --http2 https://example.com/ 2>&1 | grep -iE "HTTP/2 (103|200)"
```
### Location and redirect verification
```bash
# Single redirect
curl -sI https://example.com/old-url | grep -iE "^(HTTP|Location):"
# Full chain
curl -sLI https://example.com/old-url | grep -iE "^(HTTP|Location):"
# Count hops
HOPS=$(curl -sLI https://example.com/old-url | grep -c "^HTTP/")
echo "Hops: $HOPS"
# Final URL and status
curl -sILo /dev/null -w "Final: %{url_effective} Status: %{http_code} Redirects: %{num_redirects}\n" \
https://example.com/old-url
# Audit a list of old URLs
while IFS= read -r url; do
FINAL=$(curl -sILo /dev/null -w "%{url_effective}" "$url")
HOPS=$(curl -sLI "$url" | grep -c "^HTTP/")
STATUS=$(curl -sILo /dev/null -w "%{http_code}" "$url")
echo "$url -> $FINAL ($HOPS hops, final $STATUS)"
done < urls-to-audit.txt
# Verify POST is preserved on 307/308
curl -X POST -sI https://example.com/api/v1/something | grep -iE "^(HTTP|Location):"
# Detect redirect loop (timeout after 5 seconds, max 10 redirects)
curl -sLI --max-redirs 10 --max-time 5 https://example.com/maybe-looping
```
### Server side investigation
```bash
# Find every X-Robots-Tag in the config
grep -rn "X-Robots-Tag" /etc/nginx/
# Find every Link header
grep -rn "Link" /etc/nginx/sites-available/
# Find every redirect (return 301/302/etc, rewrite ... permanent)
grep -rn -E "return (301|302|303|307|308)|rewrite.*(permanent|redirect)" /etc/nginx/sites-available/
# Find any open redirect patterns (variable in return target)
grep -rn -E "return (301|302|303|307|308).*\\\$" /etc/nginx/sites-available/
# Check active config
nginx -T 2>/dev/null | grep -iE "x-robots-tag|^[[:space:]]*link |return 30|early_hints"
# Reload after changes
nginx -t && systemctl reload nginx
```
### Google Search Console verification
The authoritative SEO check. For each URL you care about:
1. Open Search Console for the property.
2. Top search bar: paste the URL.
3. Click Enter.
4. Wait for the report to load. Sections:
* **URL is on Google**: indexed correctly.
* **URL is not on Google**: not indexed; reason shown.
5. Click "Test live URL" to fetch with Googlebot now and see headers.
6. Under "Page indexing":
* "Indexing allowed?" should be "Yes" for pages you want indexed.
* "User-declared canonical" and "Google-selected canonical" should match.
7. Under "Crawl":
* "Last crawl" date should be recent.
* "Crawled as" should be Googlebot Smartphone (mobile first).
This is ground truth. Any disagreement between curl output and Search Console means the URL has not been re crawled yet.
---
## 14. CROSS-REFERENCES
* [framework-http-caching-headers.md](framework-http-caching-headers.md): Cache-Control, ETag, Last-Modified, Expires, Vary, Age. Preloaded resources should have correct cache directives; redirects often need no caching; X-Robots-Tag has no caching implications.
* [framework-http-content-headers.md](framework-http-content-headers.md): Content-Type, Content-Language, Content-Encoding, Content-Length, Content-Disposition. Content-Language pairs with hreflang declared via Link header. Content-Disposition pairs with X-Robots-Tag for downloadable file handling.
* [UNIVERSAL-RANKING-FRAMEWORK.md](UNIVERSAL-RANKING-FRAMEWORK.md): the master ranking reference. Section 24 (Bubbles Nginx config) inherits SEO header configuration from here.
* [SEO-BUILD-REFERENCE.md](SEO-BUILD-REFERENCE.md) v2.4: the build playbook.
* [framework-hreflang.md](framework-hreflang.md): full coverage of multi region SEO. Companion to Link header hreflang in Section 6 here.
* [framework-robotstxt-crawlbudget.md](framework-robotstxt-crawlbudget.md): robots.txt syntax and crawl budget management. Pairs with X-Robots-Tag distinction in Section 5.5.
* [framework-canonicalization.md](framework-canonicalization.md): canonical URL strategy. Companion to Link rel=canonical and the redirect/canonical interaction in Section 8.
* [framework-corewebvitals.md](framework-corewebvitals.md): LCP, INP, CLS targets. Preload via Link header and Early Hints (103) directly improve LCP.
* [framework-security-headers.md](framework-security-headers.md): HSTS, CSP, X-Frame-Options. Open redirect security pairs with the broader security header stack.
* Google X-Robots-Tag documentation: https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag
* Google redirects documentation: https://developers.google.com/search/docs/crawling-indexing/301-redirects
* Google international and multi region: https://developers.google.com/search/docs/specialty/international/overview
* MDN Link header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link
* MDN Location header: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Location
* RFC 8288 (Web Linking): https://www.rfc-editor.org/rfc/rfc8288
* RFC 9110 (HTTP Semantics, redirects and Location): https://www.rfc-editor.org/rfc/rfc9110
* HTTP 103 Early Hints: https://www.rfc-editor.org/rfc/rfc8297
---
## APPENDIX A: ONE PAGE QUICK REFERENCE
### X-Robots-Tag
Production HTML default:
```
X-Robots-Tag: max-snippet:200, max-image-preview:large, max-video-preview:-1
```
Block from indexing:
```
X-Robots-Tag: noindex, nofollow
```
Time limited:
```
X-Robots-Tag: unavailable_after: 31 Dec 2026 23:59:59 GMT
```
Bot specific:
```
X-Robots-Tag: googlebot: nosnippet
X-Robots-Tag: bingbot: noarchive
```
### Link
Canonical for non HTML:
```
Link: <https://example.com/canonical>; rel="canonical"
```
Preload hero image (LCP):
```
Link: </images/hero.webp>; rel="preload"; as="image"; fetchpriority="high"
```
Preload font:
```
Link: </fonts/main.woff2>; rel="preload"; as="font"; crossorigin
```
Hreflang (reciprocal cluster, set on every variant):
```
Link: <https://example.com/>; rel="alternate"; hreflang="en-US", <https://example.com/es/>; rel="alternate"; hreflang="es-MX", <https://example.com/>; rel="alternate"; hreflang="x-default"
```
### Location
| Scenario | Code |
|---|---|
| Permanent URL change | 301 |
| Temporary URL change | 302 |
| After form POST (force GET on next) | 303 |
| Temporary, preserve POST | 307 |
| Permanent, preserve POST | 308 |
HTTP to HTTPS:
```nginx
return 301 https://$host$request_uri;
```
www to non www:
```nginx
return 301 https://example.com$request_uri;
```
Old URL pattern to new:
```nginx
location ~ ^/old/(.*)$ { return 301 /new/$1; }
```
Three commands every operator should know:
```bash
# Confirm SEO headers on a URL
curl -sI https://example.com/page.html | grep -iE "^(x-robots-tag|link|location):"
# Trace a redirect chain
curl -sLI https://example.com/old-url | grep -iE "^(HTTP|Location):"
# Apply nginx changes
nginx -t && systemctl reload nginx
```
Two end to end tests:
```bash
# Verify a noindex directive is in effect
curl -sI -A "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" \
https://example.com/staging-page | grep -i x-robots-tag
# Should show: x-robots-tag: noindex, nofollow
# Verify a redirect resolves in one hop
curl -sILo /dev/null -w "Hops: %{num_redirects} Final: %{url_effective} Status: %{http_code}\n" \
https://example.com/old-url
# Should show: Hops: 1 Final: https://example.com/new-url Status: 200
```
If both produce expected output, the SEO header stack is correctly wired.
---
End of framework-http-seo-headers.md.