#\!/bin/bash
export PATH="/opt/homebrew/bin:$PATH"
# =============================================================================
# SEFERIM FULL 500+ API LEARNER v2
# Uses ALL available knowledge sources correctly
# =============================================================================

source ~/seferim/scripts/knowledge-sources.sh

LOG=~/.seferim/logs/full500.log
mkdir -p ~/.seferim/logs

log() { echo "[FULL-500] $1" | tee -a "$LOG"; }

persist() {
    ~/seferim/scripts/learn-persist-mb2.sh "$1" "${2:-0.8}" 2>/dev/null
}

# Sources that need a topic argument
TOPIC_SOURCES=(
    "fetch_wikipedia:epona"
    "fetch_wikidata:epona"
    "fetch_openlibrary:epona"
    "fetch_dictionary:lashon"
    "fetch_datamuse:lashon"
    "fetch_arxiv:mnemosyne"
    "fetch_pubmed:mnemosyne"
    "fetch_semantic_scholar:mnemosyne"
    "fetch_crossref:mnemosyne"
    "fetch_github:yad"
    "fetch_stackoverflow:yad"
    "fetch_npm:yad"
    "fetch_pypi:yad"
    "fetch_crates:yad"
    "fetch_duckduckgo:epona"
    "fetch_gbif:epona"
    "fetch_inaturalist:epona"
    "fetch_chembl:mnemosyne"
    "fetch_clinicaltrials:mnemosyne"
    "fetch_openalex:mnemosyne"
    "fetch_reddit:hermes"
)

# Sources that return random content (no topic needed)
RANDOM_SOURCES=(
    "fetch_hackernews:hermes"
    "fetch_hackernews_new:hermes"
    "fetch_lobsters:hermes"
    "fetch_devto:hermes"
    "fetch_spaceflight_news:selene"
    "fetch_nasa_apod:selene"
    "fetch_spacex:selene"
    "fetch_usgs_earthquakes:aurora"
    "fetch_github_trending:yad"
    "fetch_arxiv_random:mnemosyne"
    "fetch_gutenberg:euterpe"
    "fetch_quotable:euterpe"
    "fetch_zenquotes:pneuma"
    "fetch_poetrydb:euterpe"
    "fetch_trivia:athena"
    "fetch_trivia_api:athena"
    "fetch_numbersapi:athena"
    "fetch_numbersapi_math:athena"
    "fetch_uselessfact:iris"
    "fetch_catfact:iris"
    "fetch_jokeapi:iris"
    "fetch_adviceslip:pneuma"
    "fetch_cocktaildb:hestia"
    "fetch_mealdb:hestia"
    "fetch_restcountries:saga"
    "fetch_worldtimeapi:derekh"
    "fetch_coingecko:entrep"
    "fetch_coincap:entrep"
    "fetch_exchangerate:entrep"
    "fetch_openfda:hestia"
    "fetch_nba:rhiannon"
    "fetch_f1:rhiannon"
)

# Topics for topic-based sources
TOPICS=(
    "machine_learning" "quantum_computing" "neural_networks" "consciousness"
    "distributed_systems" "cryptography" "compiler_design" "neuroscience"
    "quantum_physics" "molecular_biology" "climate_science" "astrophysics"
    "genetics" "epistemology" "philosophy_of_mind" "ethics" "metaphysics"
    "topology" "graph_theory" "emergence" "robotics" "cognitive_science"
    "astronomy" "chemistry" "mathematics" "physics" "biology" "python"
    "rust" "javascript" "golang" "linux" "database" "kubernetes" "docker"
    "artificial_intelligence" "deep_learning" "reinforcement_learning"
)

pick_topic() {
    local idx=$((RANDOM % ${#TOPICS[@]}))
    echo "${TOPICS[$idx]}"
}

log "=== SEFERIM FULL 500+ API Learner v2 Starting ==="
log "Topic sources: ${#TOPIC_SOURCES[@]}"
log "Random sources: ${#RANDOM_SOURCES[@]}"
log "Total: $((${#TOPIC_SOURCES[@]} + ${#RANDOM_SOURCES[@]})) APIs"

cycle=0
while true; do
    cycle=$((cycle + 1))
    log ""
    log "═══════════════════════════════════════════"
    log "Cycle $cycle - $(date +%H:%M:%S)"
    log "═══════════════════════════════════════════"
    
    success=0
    
    # Fetch from 2 topic-based sources
    for i in 1 2; do
        idx=$((RANDOM % ${#TOPIC_SOURCES[@]}))
        source_entry="${TOPIC_SOURCES[$idx]}"
        func_name=$(echo "$source_entry" | cut -d: -f1)
        family=$(echo "$source_entry" | cut -d: -f2)
        topic=$(pick_topic)
        
        log "[$i] $func_name -> $topic"
        content=$($func_name "$topic" 2>/dev/null)
        
        if [ -n "$content" ] && [ ${#content} -gt 30 ]; then
            persist "[$family:${func_name#fetch_}] $topic: $content" 0.85
            log "    ✓ ${#content} chars"
            ((success++))
        else
            log "    ✗ No content"
        fi
        sleep 1
    done
    
    # Fetch from 3 random sources
    for i in 1 2 3; do
        idx=$((RANDOM % ${#RANDOM_SOURCES[@]}))
        source_entry="${RANDOM_SOURCES[$idx]}"
        func_name=$(echo "$source_entry" | cut -d: -f1)
        family=$(echo "$source_entry" | cut -d: -f2)
        
        log "[$((i+2))] $func_name (random)"
        content=$($func_name 2>/dev/null)
        
        if [ -n "$content" ] && [ ${#content} -gt 20 ]; then
            persist "[$family:${func_name#fetch_}] $content" 0.8
            log "    ✓ ${#content} chars"
            ((success++))
        else
            log "    ✗ No content"
        fi
        sleep 1
    done
    
    log "Cycle $cycle: $success/5 successful"
    sleep 25
done