Janady13/seo-sidecar / megamind/seo/schema_change_detector.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/seo-sidecar / megamind/seo/schema_change_detector.go
Date	Not specified in source
Repository	Janady13/seo-sidecar
Revision	c66ba23646cf638023e8803dc201dd69089107c9
Branch	main
Path or archive member	megamind/seo/schema_change_detector.go
Original SHA-256	31839415701b36292eea776e8de9c7cb9de50088db2633e7ab0a5059e7954e5e
Git blob SHA	4d4ed9dfc7be75317012365da514c52bb85bd534
Reading-copy SHA-256	31839415701b36292eea776e8de9c7cb9de50088db2633e7ab0a5059e7954e5e

Original source ↗

Recovered source transcript

Download complete source text ↓241 lines · 5.1 KB

package seo

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"time"
)

// ChangeDetector detects pages that need schema updates.
type ChangeDetector interface {
	DetectChanges() ([]PageObservation, error)
}

// MultiSourceDetector combines multiple detection strategies.
type MultiSourceDetector struct {
	registry   *SiteRegistry
	cache      *StateCache
	httpClient *http.Client
	detectors  []ChangeDetector
}

// NewMultiSourceDetector creates a detector with multiple sources.
func NewMultiSourceDetector(registry *SiteRegistry, cache *StateCache) *MultiSourceDetector {
	return &MultiSourceDetector{
		registry: registry,
		cache:    cache,
		httpClient: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

// DetectChanges aggregates changes from all sources.
func (d *MultiSourceDetector) DetectChanges() ([]PageObservation, error) {
	var changes []PageObservation

	for _, detector := range d.detectors {
		found, err := detector.DetectChanges()
		if err != nil {
			// Log but continue with other detectors
			continue
		}
		changes = append(changes, found...)
	}

	return d.deduplicate(changes), nil
}

func (d *MultiSourceDetector) deduplicate(obs []PageObservation) []PageObservation {
	seen := make(map[string]bool)
	var result []PageObservation
	for _, o := range obs {
		if !seen[o.URL] {
			seen[o.URL] = true
			result = append(result, o)
		}
	}
	return result
}

// SitemapDetector detects changes via sitemap comparison.
type SitemapDetector struct {
	registry   *SiteRegistry
	cache      *StateCache
	httpClient *http.Client
}

func NewSitemapDetector(registry *SiteRegistry, cache *StateCache) *SitemapDetector {
	return &SitemapDetector{
		registry: registry,
		cache:    cache,
		httpClient: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

func (d *SitemapDetector) DetectChanges() ([]PageObservation, error) {
	var changes []PageObservation

	sites := d.registry.ListSites()
	for _, site := range sites {
		if !d.supportsMode(site, "sitemap") {
			continue
		}

		sitemapURL := site.BaseURL + "/sitemap.xml"
		urls, err := d.parseSitemap(sitemapURL)
		if err != nil {
			continue
		}

		for _, pageURL := range urls {
			hash, err := d.fetchPageHash(pageURL)
			if err != nil {
				continue
			}

			cached := d.cache.Get(pageURL)
			if cached == nil || cached.LastContentHash != hash {
				changes = append(changes, PageObservation{
					URL:         pageURL,
					SiteKey:     site.SiteKey,
					ContentHash: hash,
					LastSeen:    time.Now(),
				})
			}
		}
	}

	return changes, nil
}

func (d *SitemapDetector) supportsMode(site SiteProfile, mode string) bool {
	for _, m := range site.ChangeDetectionModes {
		if m == mode {
			return true
		}
	}
	return false
}

func (d *SitemapDetector) parseSitemap(url string) ([]string, error) {
	// TODO: Implement sitemap XML parsing
	return nil, nil
}

func (d *SitemapDetector) fetchPageHash(url string) (string, error) {
	resp, err := d.httpClient.Get(url)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	hash := sha256.Sum256(body)
	return hex.EncodeToString(hash[:]), nil
}

// PageHashDetector detects changes via direct page hash comparison.
type PageHashDetector struct {
	registry   *SiteRegistry
	cache      *StateCache
	httpClient *http.Client
	pageURLs   []string
}

func NewPageHashDetector(registry *SiteRegistry, cache *StateCache, urls []string) *PageHashDetector {
	return &PageHashDetector{
		registry: registry,
		cache:    cache,
		httpClient: &http.Client{
			Timeout: 30 * time.Second,
		},
		pageURLs: urls,
	}
}

func (d *PageHashDetector) DetectChanges() ([]PageObservation, error) {
	var changes []PageObservation

	for _, url := range d.pageURLs {
		hash, err := d.fetchHash(url)
		if err != nil {
			continue
		}

		cached := d.cache.Get(url)
		if cached == nil || cached.LastContentHash != hash {
			site := d.registry.GetBySiteURL(url)
			siteKey := ""
			if site != nil {
				siteKey = site.SiteKey
			}

			changes = append(changes, PageObservation{
				URL:         url,
				SiteKey:     siteKey,
				ContentHash: hash,
				LastSeen:    time.Now(),
			})
		}
	}

	return changes, nil
}

func (d *PageHashDetector) fetchHash(url string) (string, error) {
	resp, err := d.httpClient.Get(url)
	if err != nil {
		return "", fmt.Errorf("fetch: %w", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("read: %w", err)
	}

	hash := sha256.Sum256(body)
	return hex.EncodeToString(hash[:]), nil
}

// WebhookDetector receives push notifications about changes.
type WebhookDetector struct {
	pending chan PageObservation
}

func NewWebhookDetector() *WebhookDetector {
	return &WebhookDetector{
		pending: make(chan PageObservation, 100),
	}
}

func (d *WebhookDetector) Push(obs PageObservation) {
	select {
	case d.pending <- obs:
	default:
		// Queue full, drop oldest
	}
}

func (d *WebhookDetector) DetectChanges() ([]PageObservation, error) {
	var changes []PageObservation
	for {
		select {
		case obs := <-d.pending:
			changes = append(changes, obs)
		default:
			return changes, nil
		}
	}
}