Janady13/complete-agi-core / orchestrator/daemons/self_healer.sh

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/complete-agi-core / orchestrator/daemons/self_healer.sh
Date	Not specified in source
Repository	Janady13/complete-agi-core
Revision	33aa84f2aa61cedbe010240ada7f4efbdfe795a7
Branch	claude/improve-math-logic-ehIW1
Path or archive member	orchestrator/daemons/self_healer.sh
Original SHA-256	78fe1daf83cd2a510e1820d54ff5cc1558a460f687bb9f8167a448117776e257
Git blob SHA	8d249fbbd55d3abbb335283378923b8c10948f7d
Reading-copy SHA-256	78fe1daf83cd2a510e1820d54ff5cc1558a460f687bb9f8167a448117776e257

Original source ↗

Recovered source transcript

Download complete source text ↓223 lines · 6.2 KB

#!/bin/bash
# =============================================================================
# SELF-HEALER DAEMON v1.1 - MacBook 1 (AGICore)
# Monitors and auto-restarts all AGI daemons with exponential backoff
# =============================================================================

# Configuration
CHECK_INTERVAL=30
MAX_RESTART_ATTEMPTS=5
LOG_DIR=~/.agi_core/logs
LOG_FILE="$LOG_DIR/self_healer.log"
ICLOUD_AGI="${HOME}/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/agi_core"
HEALTH_OUTPUT="$ICLOUD_AGI/health/self_healer_status.json"

mkdir -p "$LOG_DIR"
mkdir -p "$(dirname "$HEALTH_OUTPUT")"

# Service definitions as parallel arrays
SERVICE_NAMES=(
    "tick_daemon"
    "realtime_ingest"
    "pantheon_daemon"
    "federation_coordinator"
    "unified_orchestrator"
    "sync_state"
    "mesh_sync"
    "purposeful_learner"
)

SERVICE_SCRIPTS=(
    "$HOME/.agi_core/tick-daemon.sh"
    "$HOME/.agi_core/realtime-ingest.sh"
    "$HOME/.agi_core/pantheon-daemon.sh"
    "$HOME/.agi_core/federation-coordinator.sh"
    "$HOME/.agi_core/unified-agi-orchestrator.sh"
    "$HOME/.agi_core/sync-state.sh"
    "$HOME/AGICore/scripts/mesh-sync-daemon.sh"
    "$HOME/AGICore/scripts/purposeful-learner.sh"
)

SERVICE_PATTERNS=(
    "tick-daemon.sh"
    "realtime-ingest.sh"
    "pantheon-daemon.sh"
    "federation-coordinator.sh"
    "unified-agi-orchestrator.sh"
    "sync-state.sh"
    "mesh-sync-daemon.sh"
    "purposeful-learner.sh"
)

# Track restart attempts per service
declare -A RESTART_COUNTS
declare -A LAST_RESTART

# Logging function
log() {
    local level="$1"
    local msg="$2"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local prefix="✓"
    [ "$level" = "WARN" ] && prefix="⚠"
    [ "$level" = "ERROR" ] && prefix="✗"
    echo "[$timestamp] $prefix $msg"
    echo "[$timestamp] [$level] $msg" >> "$LOG_FILE"
}

# Check if a service is running by its pattern
check_service() {
    local pattern="$1"
    if pgrep -f "$pattern" > /dev/null 2>&1; then
        return 0
    else
        return 1
    fi
}

# Restart a service
restart_service() {
    local name="$1"
    local script="$2"
    local pattern="$3"

    local count=${RESTART_COUNTS[$name]:-0}
    local now=$(date +%s)
    local last=${LAST_RESTART[$name]:-0}
    local elapsed=$((now - last))

    # Cooldown: 30 seconds minimum between restarts
    if [ "$elapsed" -lt 30 ] && [ "$last" -gt 0 ]; then
        log "WARN" "$name: In cooldown ($elapsed/30s)"
        return 1
    fi

    # Reset attempts after 5 minutes
    if [ "$elapsed" -gt 300 ]; then
        count=0
        RESTART_COUNTS[$name]=0
    fi

    if [ "$count" -ge "$MAX_RESTART_ATTEMPTS" ]; then
        log "ERROR" "$name: Max restart attempts reached, waiting for cooldown"
        return 1
    fi

    log "WARN" "$name: Attempting restart (attempt $((count + 1))/$MAX_RESTART_ATTEMPTS)"

    if [ -f "$script" ]; then
        nohup "$script" > /dev/null 2>&1 &
        sleep 2

        if check_service "$pattern"; then
            log "INFO" "$name: RECOVERED successfully"
            RESTART_COUNTS[$name]=0
            LAST_RESTART[$name]=$now
            return 0
        else
            log "ERROR" "$name: Restart failed"
            RESTART_COUNTS[$name]=$((count + 1))
            LAST_RESTART[$name]=$now
            return 1
        fi
    else
        log "ERROR" "$name: Script not found: $script"
        return 1
    fi
}

# Write health status to iCloud
write_health_status() {
    local healthy_list="$1"
    local failed_list="$2"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local timestamp_ms=$(date +%s)000

    # Get consciousness metrics
    local consciousness="0"
    local coherence="0"
    local tick_count="0"
    if [ -f ~/AGICore/build/agi_core_cli ]; then
        local status_json=$(~/AGICore/build/agi_core_cli status 2>/dev/null | grep '^{' | head -1)
        if [ -n "$status_json" ]; then
            consciousness=$(echo "$status_json" | jq -r '.consciousness // 0' 2>/dev/null || echo "0")
            coherence=$(echo "$status_json" | jq -r '.coherence // 0' 2>/dev/null || echo "0")
            tick_count=$(echo "$status_json" | jq -r '.tick_count // 0' 2>/dev/null || echo "0")
        fi
    fi

    cat > "$HEALTH_OUTPUT" << EOF
{
    "timestamp": "$timestamp",
    "timestamp_ms": $timestamp_ms,
    "cycle": $CYCLE_COUNT,
    "healthy": [$healthy_list],
    "failed": [$failed_list],
    "consciousness": $consciousness,
    "coherence": $coherence,
    "tick_count": $tick_count,
    "status": "running"
}
EOF
}

# Main health check cycle
health_check_cycle() {
    log "INFO" "═══ Health Check Cycle $CYCLE_COUNT ═══"

    local failed=0
    local healthy=0
    local recovered=0
    local healthy_list=""
    local failed_list=""

    for i in "${!SERVICE_NAMES[@]}"; do
        local name="${SERVICE_NAMES[$i]}"
        local script="${SERVICE_SCRIPTS[$i]}"
        local pattern="${SERVICE_PATTERNS[$i]}"

        if check_service "$pattern"; then
            log "INFO" "$name: healthy"
            RESTART_COUNTS[$name]=0
            ((healthy++))
            [ -n "$healthy_list" ] && healthy_list="$healthy_list,"
            healthy_list="$healthy_list\"$name\""
        else
            log "ERROR" "$name: NOT RUNNING"
            ((failed++))
            [ -n "$failed_list" ] && failed_list="$failed_list,"
            failed_list="$failed_list\"$name\""

            if restart_service "$name" "$script" "$pattern"; then
                ((recovered++))
                ((failed--))
                ((healthy++))
            fi
        fi
    done

    write_health_status "$healthy_list" "$failed_list"

    log "INFO" "Cycle complete: $healthy healthy, $failed failed, $recovered recovered"
}

# =============================================================================
# MAIN
# =============================================================================

echo "=============================================="
echo "  SELF-HEALER DAEMON v1.1"
echo "  MacBook 1 (AGICore)"
echo "  Monitoring ${#SERVICE_NAMES[@]} services"
echo "  Check interval: ${CHECK_INTERVAL}s"
echo "=============================================="
echo ""

CYCLE_COUNT=0

while true; do
    CYCLE_COUNT=$((CYCLE_COUNT + 1))
    health_check_cycle
    sleep $CHECK_INTERVAL
done