Janady13/llms-txt-hub / apps/web/app/api/fetch-metadata/__tests__/route.test.ts

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/llms-txt-hub / apps/web/app/api/fetch-metadata/__tests__/route.test.ts
Date	Not specified in source
Repository	Janady13/llms-txt-hub
Revision	e0f4bf978d4df168369c7601d9c5f5ea89e84377
Branch	main
Path or archive member	apps/web/app/api/fetch-metadata/__tests__/route.test.ts
Original SHA-256	21cc79d43bed06c31e8b78a9739925bc8326995f659d58e07f146f4ad4d1a8aa
Git blob SHA	b09db5714bbe3e0a0d7fbb955a52ad7e3f612e49
Reading-copy SHA-256	21cc79d43bed06c31e8b78a9739925bc8326995f659d58e07f146f4ad4d1a8aa
Original source ↗
Recovered source transcript

Download complete source text ↓457 lines · 14.6 KB
/**
 * Tests for fetch-metadata API route
 *
 * Tests metadata extraction, URL validation, duplicate detection, and sanitization.
 */

import * as cheerio from 'cheerio'
import { GET, POST } from '@/app/api/fetch-metadata/route'
import { getWebsites } from '@/lib/content-loader'

// Mock dependencies
jest.mock('@/lib/content-loader')
jest.mock('cheerio', () => ({
  load: jest.fn(() => {
    const $ = (_selector: string) => ({
      text: jest.fn().mockReturnValue('Test Title'),
      attr: jest.fn().mockReturnValue('Test content')
    })
    return $
  })
}))
jest.mock('@thedaviddias/logging', () => ({
  logger: {
    error: jest.fn()
  }
}))

// Mock fetch globally
global.fetch = jest.fn()

describe('Fetch Metadata API Route', () => {
  const mockFetch = global.fetch as jest.MockedFunction<typeof fetch>
  const mockGetWebsites = getWebsites as jest.MockedFunction<typeof getWebsites>
  const mockCheerioLoad = cheerio.load as jest.MockedFunction<typeof cheerio.load>

  beforeEach(() => {
    jest.clearAllMocks()

    // Default mock implementations
    mockGetWebsites.mockReturnValue([])

    // Mock cheerio
    const mockCheerioInstance = {
      text: jest.fn().mockReturnValue('Test Title'),
      attr: jest.fn().mockReturnValue('Test content')
    }
    mockCheerioLoad.mockReturnValue(jest.fn().mockReturnValue(mockCheerioInstance) as any)

    // Mock successful fetch responses
    mockFetch.mockImplementation((url: string | Request | URL) => {
      const urlString = typeof url === 'string' ? url : url.toString()
      if (urlString.includes('llms.txt') || urlString.includes('llms-full.txt')) {
        return Promise.resolve({
          ok: true,
          status: 200
        } as Response)
      }
      return Promise.resolve({
        ok: true,
        status: 200,
        text: () => Promise.resolve('<html><title>Test Title</title></html>')
      } as Response)
    })
  })

  describe('GET Request', () => {
    it('returns 400 when domain is missing', async () => {
      const request = new Request('http://localhost/api/fetch-metadata')
      const response = await GET(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('Domain is required')
    })

    it('returns 400 for invalid URL format', async () => {
      const request = new Request('http://localhost/api/fetch-metadata?domain=not-a-url')
      const response = await GET(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('Invalid URL format')
    })

    it('blocks malicious URL protocols', async () => {
      const maliciousUrls = [
        'javascript:alert(1)',
        'data:text/html,<script>alert(1)</script>',
        'vbscript:msgbox'
      ]

      for (const url of maliciousUrls) {
        const request = new Request(
          `http://localhost/api/fetch-metadata?domain=${encodeURIComponent(url)}`
        )
        const response = await GET(request)
        const data = await response.json()

        expect(response.status).toBe(400)
        expect(data.error).toBe('Invalid URL protocol')
      }
    })

    it('blocks localhost and private-network targets', async () => {
      const blockedUrls = ['http://localhost:3000', 'http://127.0.0.1', 'http://192.168.1.10']

      for (const url of blockedUrls) {
        const request = new Request(
          `http://localhost/api/fetch-metadata?domain=${encodeURIComponent(url)}`
        )
        const response = await GET(request)
        const data = await response.json()

        expect(response.status).toBe(400)
        expect(data.error).toBe('URL points to a restricted network address')
      }
    })

    it('successfully fetches metadata for valid URL', async () => {
      const request = new Request('http://localhost/api/fetch-metadata?domain=https://example.com')
      const response = await GET(request)
      const data = await response.json()

      expect(response.status).toBe(200)
      expect(data.metadata).toBeDefined()
      expect(data.metadata.name).toBeDefined()
      expect(data.metadata.website).toBe('https://example.com/')
    })

    it('blocks redirects to restricted network targets', async () => {
      mockFetch.mockResolvedValueOnce(
        new Response(null, {
          status: 302,
          headers: {
            location: 'http://127.0.0.1/private'
          }
        })
      )

      const request = new Request('http://localhost/api/fetch-metadata?domain=https://example.com')
      const response = await GET(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('URL points to a restricted network address')
    })
  })

  describe('POST Request', () => {
    it('returns 400 when website is missing', async () => {
      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({})
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('Website URL is required')
    })

    it('returns 400 when website is not a string', async () => {
      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 123 })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('Website URL is required')
    })

    it('returns 400 for invalid URL format', async () => {
      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'not-a-url' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('Invalid URL format')
    })

    it('blocks malicious URL protocols', async () => {
      const maliciousUrls = [
        'javascript:alert(1)',
        'data:text/html,<script>alert(1)</script>',
        'vbscript:msgbox'
      ]

      for (const url of maliciousUrls) {
        const request = new Request('http://localhost/api/fetch-metadata', {
          method: 'POST',
          headers: { 'Content-Type': 'application/json' },
          body: JSON.stringify({ website: url })
        })

        const response = await POST(request)
        const data = await response.json()

        expect(response.status).toBe(400)
        expect(data.error).toBe('Invalid URL protocol')
      }
    })

    it('blocks localhost and private-network targets', async () => {
      const blockedUrls = ['http://localhost:3000', 'http://127.0.0.1', 'http://192.168.1.10']

      for (const website of blockedUrls) {
        const request = new Request('http://localhost/api/fetch-metadata', {
          method: 'POST',
          headers: { 'Content-Type': 'application/json' },
          body: JSON.stringify({ website })
        })

        const response = await POST(request)
        const data = await response.json()

        expect(response.status).toBe(400)
        expect(data.error).toBe('URL points to a restricted network address')
      }
    })

    it('successfully fetches metadata for valid URL', async () => {
      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(200)
      expect(data.metadata).toBeDefined()
      expect(data.metadata.name).toBeDefined()
      expect(data.metadata.website).toBe('https://example.com/')
    })

    it('blocks redirects to restricted network targets', async () => {
      mockFetch.mockResolvedValueOnce(
        new Response(null, {
          status: 302,
          headers: {
            location: 'http://localhost:8080/internal'
          }
        })
      )

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(400)
      expect(data.error).toBe('URL points to a restricted network address')
    })

    it('handles duplicate website detection', async () => {
      mockGetWebsites.mockReturnValue([
        {
          name: 'Existing Site',
          description: 'An existing website',
          website: 'https://example.com',
          category: 'Tools',
          slug: 'existing-site',
          llmsUrl: 'https://example.com/llms.txt',
          publishedAt: '2024-01-01'
        }
      ])

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(200)
      expect(data.isDuplicate).toBe(true)
      expect(data.existingWebsite).toBeDefined()
    })

    it('handles fetch errors gracefully', async () => {
      mockFetch.mockRejectedValueOnce(new Error('Network error'))

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(500)
      expect(data.error).toBe('Failed to fetch metadata')
    })

    it('handles non-200 responses', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: false,
        status: 404
      } as Response)

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(500)
      expect(data.error).toBe('Failed to fetch metadata')
    })

    it('sanitizes HTML content', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        status: 200,
        text: () =>
          Promise.resolve('<html><title>Test Title<script>alert("xss")</script></title></html>')
      } as Response)

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(200)
      // Script content should have been stripped by stripHtml
      if (data.metadata?.name) {
        expect(data.metadata.name).not.toContain('<script')
        expect(data.metadata.name).not.toContain('alert')
      }
    })

    it('extracts metadata from HTML', async () => {
      const htmlContent = `
        <html>
          <head>
            <title>Test Website</title>
            <meta name="description" content="Test description">
            <meta property="og:title" content="OG Title">
            <meta property="og:description" content="OG Description">
            <meta property="og:image" content="https://example.com/og-image.jpg">
            <link rel="icon" href="https://example.com/favicon.ico">
          </head>
          <body>
            <h1>Test Website</h1>
          </body>
        </html>
      `

      // Mock cheerio to return the actual content from the HTML
      const mockCheerioInstance = {
        text: jest.fn().mockReturnValue('Test Website'),
        attr: jest.fn().mockImplementation((attr: string) => {
          if (attr === 'content') {
            return 'Test description'
          }
          return null
        })
      }
      mockCheerioLoad.mockReturnValueOnce(jest.fn().mockReturnValue(mockCheerioInstance) as any)

      mockFetch.mockResolvedValueOnce({
        ok: true,
        status: 200,
        text: () => Promise.resolve(htmlContent)
      } as Response)

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(200)
      expect(data.metadata.name).toBe('Test Website')
      expect(data.metadata.description).toBe('Test description')
      expect(data.metadata.website).toBe('https://example.com/')
    })

    it('handles missing metadata gracefully', async () => {
      const htmlContent = '<html><body><h1>No metadata</h1></body></html>'

      mockFetch.mockResolvedValueOnce({
        ok: true,
        status: 200,
        text: () => Promise.resolve(htmlContent)
      } as Response)

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(200)
      expect(data.metadata.name).toBeDefined()
      expect(data.metadata.description).toBeDefined()
      expect(data.metadata.website).toBe('https://example.com/')
    })

    it('handles cheerio load errors', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        status: 200,
        text: () => Promise.resolve('<html><title>Test</title></html>')
      } as Response)

      mockCheerioLoad.mockImplementationOnce(() => {
        throw new Error('Cheerio load error')
      })

      const request = new Request('http://localhost/api/fetch-metadata', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({ website: 'https://example.com' })
      })

      const response = await POST(request)
      const data = await response.json()

      expect(response.status).toBe(500)
      expect(data.error).toBe('Failed to fetch metadata')
    })
  })
})