Janady13/AI_Brain / the_tired_agi/scripts/train_with_api.sh

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/AI_Brain / the_tired_agi/scripts/train_with_api.sh
Date	Not specified in source
Repository	Janady13/AI_Brain
Revision	d6c309d3fa439714ca46b9d3bb94caf478ea5376
Branch	main
Path or archive member	the_tired_agi/scripts/train_with_api.sh
Original SHA-256	44e96bbb664d875294068e17cc2baa0f6e7705c912913f739d068125e19e62ee
Git blob SHA	169f1de4ef869e593f747f4040b2c38577c93b05
Reading-copy SHA-256	44e96bbb664d875294068e17cc2baa0f6e7705c912913f739d068125e19e62ee

Original source ↗

Every recorded occurrence

These 2 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/AI_Brain / the_tired_agi/scripts/train_with_api.sh	main	d6c309d3fa439714ca46b9d3bb94caf478ea5376	Open original
Janady13/AI_Brain / the_tired_agi/scripts/train_with_api.sh	claude/binary-helix-fibonacci-hive-011CUj7dx1WWJLVYcWCjBbYv	527d4882c71eb041393f9d8945996177c5ac9e86	Open original

Recovered source transcript

Download complete source text ↓179 lines · 5.9 KB

#!/bin/bash
# AGI API-Enhanced Training
# Uses API to generate high-quality training data for natural language understanding

set -e

# Source environment
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/train_init.sh"

# API Configuration
API_ENDPOINT="https://api.anthropic.com/v1/messages"
API_MODEL="claude-3-5-sonnet-20241022"

# Paths
CHAT_BIN="$AGI_HOME/core/build/bin/agi_chat"
API_TRAINING_DATA="$TRAINING_DATA_DIR/api_training_data.jsonl"
ENHANCED_CORPUS="$TRAINING_DATA_DIR/enhanced_corpus.txt"

mkdir -p "$TRAINING_DATA_DIR"

log "───────────────────────────────────────────────"
log "🤖 API-ENHANCED TRAINING PIPELINE"
log "───────────────────────────────────────────────"
log "API Endpoint: $API_ENDPOINT"
log "Model: $API_MODEL"
log "Credits Available: $((MAX_CREDITS - CREDITS_USED))"
log ""

# Function to call Claude API
call_api() {
    local prompt="$1"
    local response_file="$2"

    # Construct API request
    local request_json=$(cat <<EOF
{
  "model": "$API_MODEL",
  "max_tokens": 1024,
  "messages": [
    {
      "role": "user",
      "content": "$prompt"
    }
  ]
}
EOF
)

    # Call API
    local http_code=$(curl -s -w "%{http_code}" -o "$response_file" \
        -X POST "$API_ENDPOINT" \
        -H "Content-Type: application/json" \
        -H "x-api-key: $API_KEY" \
        -H "anthropic-version: 2023-06-01" \
        -d "$request_json")

    if [ "$http_code" = "200" ]; then
        CREDITS_USED=$((CREDITS_USED + 1))
        log "   API call successful (Credits: $CREDITS_USED/$MAX_CREDITS)"
        return 0
    else
        log "   ⚠️  API call failed (HTTP $http_code)"
        return 1
    fi
}

# Training topics for generating Q&A pairs
declare -a TRAINING_TOPICS=(
    "Explain how distributed agent systems work in simple terms"
    "What is Fibonacci growth control in agent-based systems?"
    "Describe MAX consensus algorithm for distributed goal convergence"
    "How does binary protocol improve agent migration efficiency?"
    "Explain natural language understanding in conversational AI"
    "What is the role of memory in AI conversation systems?"
    "Describe knowledge base management in AGI systems"
    "How do hash-based phrase matching algorithms work?"
    "Explain the concept of agent swarms in distributed computing"
    "What are the benefits of P2P architecture in AGI systems?"
)

log "Generating training data using Claude API..."
log "Topics to explore: ${#TRAINING_TOPICS[@]}"
log ""

# Generate enhanced training data
for i in "${!TRAINING_TOPICS[@]}"; do
    TOPIC="${TRAINING_TOPICS[$i]}"
    SESSION=$((i + 1))

    log "Session $SESSION: Generating knowledge for \"${TOPIC:0:50}...\""

    # Check credit limit
    if [ $CREDITS_USED -ge $MAX_CREDITS ]; then
        log "⚠️  Credit limit reached ($CREDITS_USED/$MAX_CREDITS)"
        break
    fi

    # Generate Q&A pair using API
    RESPONSE_FILE=$(mktemp)
    PROMPT="Generate a clear, concise explanation (2-3 sentences) for this AGI system topic: $TOPIC"

    if call_api "$PROMPT" "$RESPONSE_FILE"; then
        # Extract response text from JSON using jq
        ANSWER=$(jq -r '.content[0].text' "$RESPONSE_FILE" 2>/dev/null || echo "API response parse error")

        if [ "$ANSWER" != "API response parse error" ] && [ -n "$ANSWER" ]; then
            # Add to training corpus
            echo "Q: $TOPIC" >> "$ENHANCED_CORPUS"
            echo "A: $ANSWER" >> "$ENHANCED_CORPUS"
            echo "" >> "$ENHANCED_CORPUS"

            # Log to JSONL for future use (escape quotes in answer for valid JSON)
            ESCAPED_ANSWER=$(echo "$ANSWER" | sed 's/"/\\"/g' | tr '\n' ' ')
            echo "{\"topic\":\"$TOPIC\",\"answer\":\"$ESCAPED_ANSWER\",\"timestamp\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" >> "$API_TRAINING_DATA"

            log "   Generated: ${ANSWER:0:80}..."
        else
            log "   Failed to parse API response"
        fi
    else
        log "   Skipped due to API error"
    fi

    rm -f "$RESPONSE_FILE"

    # Rate limiting
    sleep 1
done

log ""
log "───────────────────────────────────────────────"
log "✅ API-Enhanced Training Complete"
log "───────────────────────────────────────────────"

# Statistics
if [ -f "$ENHANCED_CORPUS" ]; then
    CORPUS_LINES=$(wc -l < "$ENHANCED_CORPUS")
    CORPUS_WORDS=$(wc -w < "$ENHANCED_CORPUS")
    CORPUS_SIZE=$(du -h "$ENHANCED_CORPUS" | cut -f1)

    log ""
    log "Enhanced Training Data Statistics:"
    log "  Lines: $CORPUS_LINES"
    log "  Words: $CORPUS_WORDS"
    log "  Size: $CORPUS_SIZE"
    log "  File: $ENHANCED_CORPUS"
fi

log ""
log "API Usage:"
log "  Credits Used: $CREDITS_USED"
log "  Credits Remaining: $((MAX_CREDITS - CREDITS_USED))"
log "  Total Budget: $MAX_CREDITS"

# Test the AGI with new knowledge
log ""
log "Testing AGI with enhanced knowledge..."
log ""

TEST_PROMPT="what is MAX consensus?"
log "Test query: $TEST_PROMPT"

RESPONSE=$(echo -e "$TEST_PROMPT\nexit" | "$CHAT_BIN" 2>/dev/null | grep "^AGI:" | head -1 | sed 's/^AGI: //')
log "AGI Response: $RESPONSE"

log ""
log "───────────────────────────────────────────────"
log "🎓 Training pipeline ready for next phase"
log "───────────────────────────────────────────────"
log ""
log "Files created:"
log "  - Enhanced corpus: $ENHANCED_CORPUS"
log "  - API training data: $API_TRAINING_DATA"
log ""
log "To continue training: $0"
log "To chat interactively: $CHAT_BIN"
log ""