Janady13/complete-agi-core / orchestrator/lib/supervisor.sh

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/complete-agi-core / orchestrator/lib/supervisor.sh
Date	Not specified in source
Repository	Janady13/complete-agi-core
Revision	33aa84f2aa61cedbe010240ada7f4efbdfe795a7
Branch	claude/improve-math-logic-ehIW1
Path or archive member	orchestrator/lib/supervisor.sh
Original SHA-256	1f12dfceee88b956888ed380b564199f511834bc3c76a1966433dd2308919f09
Git blob SHA	d5cb5162d588a3cf35af4268add156e7e673f93a
Reading-copy SHA-256	1f12dfceee88b956888ed380b564199f511834bc3c76a1966433dd2308919f09
Original source ↗
Recovered source transcript

Download complete source text ↓619 lines · 18.6 KB
#!/bin/bash
#===============================================================================
# supervisor.sh - Robust Daemon Supervision Library
#
# Provides:
#   - Automatic restart with exponential backoff
#   - Health checking with configurable probes
#   - Graceful shutdown handling
#   - State persistence and recovery
#   - Inter-daemon coordination
#   - Logging with rotation
#
# Usage:
#   source lib/supervisor.sh
#   supervisor_run "daemon_name" "command" [options]
#===============================================================================

set -euo pipefail

#===============================================================================
# CONFIGURATION
#===============================================================================

SUPERVISOR_DIR="${SUPERVISOR_DIR:-/tmp/agi-supervisor}"
SUPERVISOR_LOG_DIR="${SUPERVISOR_LOG_DIR:-$SUPERVISOR_DIR/logs}"
SUPERVISOR_STATE_DIR="${SUPERVISOR_STATE_DIR:-$SUPERVISOR_DIR/state}"
SUPERVISOR_PID_DIR="${SUPERVISOR_PID_DIR:-$SUPERVISOR_DIR/pids}"

# Restart policy
SUPERVISOR_MAX_RESTARTS="${SUPERVISOR_MAX_RESTARTS:-10}"
SUPERVISOR_RESTART_WINDOW="${SUPERVISOR_RESTART_WINDOW:-300}"  # 5 minutes
SUPERVISOR_BACKOFF_INITIAL="${SUPERVISOR_BACKOFF_INITIAL:-1}"
SUPERVISOR_BACKOFF_MAX="${SUPERVISOR_BACKOFF_MAX:-60}"
SUPERVISOR_BACKOFF_MULTIPLIER="${SUPERVISOR_BACKOFF_MULTIPLIER:-2}"

# Health check
SUPERVISOR_HEALTH_INTERVAL="${SUPERVISOR_HEALTH_INTERVAL:-30}"
SUPERVISOR_HEALTH_TIMEOUT="${SUPERVISOR_HEALTH_TIMEOUT:-10}"
SUPERVISOR_HEALTH_RETRIES="${SUPERVISOR_HEALTH_RETRIES:-3}"

# Shutdown
SUPERVISOR_SHUTDOWN_TIMEOUT="${SUPERVISOR_SHUTDOWN_TIMEOUT:-30}"

#===============================================================================
# INITIALIZATION
#===============================================================================

supervisor_init() {
    mkdir -p "$SUPERVISOR_DIR" "$SUPERVISOR_LOG_DIR" "$SUPERVISOR_STATE_DIR" "$SUPERVISOR_PID_DIR"

    # Create lock for coordination
    exec 200>"$SUPERVISOR_DIR/.lock"
}

#===============================================================================
# LOGGING
#===============================================================================

_log_file=""

supervisor_log() {
    local level="$1"
    shift
    local msg="$*"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local log_line="[$timestamp] [$level] $msg"

    echo "$log_line" >&2

    if [[ -n "$_log_file" ]]; then
        echo "$log_line" >> "$_log_file"

        # Rotate if > 10MB
        local size=$(stat -f%z "$_log_file" 2>/dev/null || stat -c%s "$_log_file" 2>/dev/null || echo 0)
        if [[ $size -gt 10485760 ]]; then
            mv "$_log_file" "$_log_file.$(date +%s)"
            # Keep only last 5 rotated logs
            ls -t "$_log_file."* 2>/dev/null | tail -n +6 | xargs rm -f 2>/dev/null || true
        fi
    fi
}

log_info()  { supervisor_log "INFO" "$@"; }
log_warn()  { supervisor_log "WARN" "$@"; }
log_error() { supervisor_log "ERROR" "$@"; }
log_debug() { [[ "${SUPERVISOR_DEBUG:-0}" == "1" ]] && supervisor_log "DEBUG" "$@" || true; }

#===============================================================================
# STATE MANAGEMENT
#===============================================================================

declare -A _daemon_state

state_save() {
    local name="$1"
    local state_file="$SUPERVISOR_STATE_DIR/${name}.state"

    cat > "$state_file" <<EOF
DAEMON_NAME="$name"
DAEMON_PID="${_daemon_state[${name}_pid]:-}"
DAEMON_START_TIME="${_daemon_state[${name}_start_time]:-0}"
DAEMON_RESTART_COUNT="${_daemon_state[${name}_restart_count]:-0}"
DAEMON_LAST_RESTART="${_daemon_state[${name}_last_restart]:-0}"
DAEMON_STATUS="${_daemon_state[${name}_status]:-unknown}"
DAEMON_PHI="${_daemon_state[${name}_phi]:-0.5}"
DAEMON_TICKS="${_daemon_state[${name}_ticks]:-0}"
DAEMON_ERRORS="${_daemon_state[${name}_errors]:-0}"
EOF
}

state_load() {
    local name="$1"
    local state_file="$SUPERVISOR_STATE_DIR/${name}.state"

    if [[ -f "$state_file" ]]; then
        source "$state_file"
        _daemon_state[${name}_pid]="$DAEMON_PID"
        _daemon_state[${name}_start_time]="$DAEMON_START_TIME"
        _daemon_state[${name}_restart_count]="$DAEMON_RESTART_COUNT"
        _daemon_state[${name}_last_restart]="$DAEMON_LAST_RESTART"
        _daemon_state[${name}_status]="$DAEMON_STATUS"
        _daemon_state[${name}_phi]="${DAEMON_PHI:-0.5}"
        _daemon_state[${name}_ticks]="${DAEMON_TICKS:-0}"
        _daemon_state[${name}_errors]="${DAEMON_ERRORS:-0}"
        return 0
    fi
    return 1
}

state_set() {
    local name="$1"
    local key="$2"
    local value="$3"
    _daemon_state[${name}_${key}]="$value"
    state_save "$name"
}

state_get() {
    local name="$1"
    local key="$2"
    echo "${_daemon_state[${name}_${key}]:-}"
}

#===============================================================================
# PROCESS MANAGEMENT
#===============================================================================

pid_write() {
    local name="$1"
    local pid="$2"
    echo "$pid" > "$SUPERVISOR_PID_DIR/${name}.pid"
}

pid_read() {
    local name="$1"
    local pid_file="$SUPERVISOR_PID_DIR/${name}.pid"
    [[ -f "$pid_file" ]] && cat "$pid_file" || echo ""
}

pid_remove() {
    local name="$1"
    rm -f "$SUPERVISOR_PID_DIR/${name}.pid"
}

process_running() {
    local pid="$1"
    [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
}

#===============================================================================
# HEALTH CHECKING
#===============================================================================

# Default health check: process exists
health_check_default() {
    local name="$1"
    local pid=$(state_get "$name" "pid")
    process_running "$pid"
}

# TCP health check
health_check_tcp() {
    local host="$1"
    local port="$2"
    local timeout="${3:-5}"

    timeout "$timeout" bash -c "cat < /dev/null > /dev/tcp/$host/$port" 2>/dev/null
}

# HTTP health check
health_check_http() {
    local url="$1"
    local timeout="${2:-5}"

    local status=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$timeout" "$url" 2>/dev/null)
    [[ "$status" == "200" ]]
}

# Custom health check (file-based)
health_check_file() {
    local name="$1"
    local health_file="$SUPERVISOR_STATE_DIR/${name}.health"
    local max_age="${2:-60}"  # Max age in seconds

    if [[ -f "$health_file" ]]; then
        local file_time=$(stat -f%m "$health_file" 2>/dev/null || stat -c%Y "$health_file" 2>/dev/null)
        local now=$(date +%s)
        local age=$((now - file_time))
        [[ $age -lt $max_age ]]
    else
        return 1
    fi
}

# Run health check with retries
run_health_check() {
    local name="$1"
    local check_fn="${2:-health_check_default}"
    local retries="${SUPERVISOR_HEALTH_RETRIES}"

    for ((i=1; i<=retries; i++)); do
        if $check_fn "$name"; then
            return 0
        fi
        log_debug "Health check failed for $name (attempt $i/$retries)"
        sleep 1
    done

    return 1
}

#===============================================================================
# RESTART POLICY
#===============================================================================

should_restart() {
    local name="$1"
    local exit_code="$2"

    # Check restart count within window
    local restart_count=$(state_get "$name" "restart_count")
    local last_restart=$(state_get "$name" "last_restart")
    local now=$(date +%s)

    # Reset counter if outside window
    if [[ $((now - last_restart)) -gt $SUPERVISOR_RESTART_WINDOW ]]; then
        restart_count=0
        state_set "$name" "restart_count" "0"
    fi

    # Check if exceeded max restarts
    if [[ $restart_count -ge $SUPERVISOR_MAX_RESTARTS ]]; then
        log_error "Daemon $name exceeded max restarts ($SUPERVISOR_MAX_RESTARTS) within window"
        return 1
    fi

    # Always restart on certain exit codes
    case $exit_code in
        0)   return 1 ;;  # Clean exit
        137) return 0 ;;  # SIGKILL (OOM, etc.)
        143) return 1 ;;  # SIGTERM (requested shutdown)
        *)   return 0 ;;  # Other errors
    esac
}

compute_backoff() {
    local restart_count="$1"
    local backoff=$SUPERVISOR_BACKOFF_INITIAL

    for ((i=0; i<restart_count; i++)); do
        backoff=$((backoff * SUPERVISOR_BACKOFF_MULTIPLIER))
        if [[ $backoff -gt $SUPERVISOR_BACKOFF_MAX ]]; then
            backoff=$SUPERVISOR_BACKOFF_MAX
            break
        fi
    done

    echo "$backoff"
}

#===============================================================================
# SIGNAL HANDLING
#===============================================================================

_shutdown_requested=0
_current_daemon_pid=""
_current_daemon_name=""

handle_shutdown() {
    log_info "Shutdown requested for $_current_daemon_name"
    _shutdown_requested=1

    if [[ -n "$_current_daemon_pid" ]] && process_running "$_current_daemon_pid"; then
        log_info "Sending SIGTERM to daemon (PID: $_current_daemon_pid)"
        kill -TERM "$_current_daemon_pid" 2>/dev/null || true

        # Wait for graceful shutdown
        local waited=0
        while process_running "$_current_daemon_pid" && [[ $waited -lt $SUPERVISOR_SHUTDOWN_TIMEOUT ]]; do
            sleep 1
            ((waited++))
        done

        # Force kill if still running
        if process_running "$_current_daemon_pid"; then
            log_warn "Daemon did not stop gracefully, sending SIGKILL"
            kill -KILL "$_current_daemon_pid" 2>/dev/null || true
        fi
    fi

    state_set "$_current_daemon_name" "status" "stopped"
    pid_remove "$_current_daemon_name"

    exit 0
}

setup_signal_handlers() {
    trap handle_shutdown SIGTERM SIGINT SIGHUP
}

#===============================================================================
# MAIN SUPERVISOR LOOP
#===============================================================================

supervisor_run() {
    local name="$1"
    local command="$2"
    shift 2
    local health_check="${1:-health_check_default}"

    supervisor_init

    _log_file="$SUPERVISOR_LOG_DIR/${name}.log"
    _current_daemon_name="$name"

    setup_signal_handlers

    # Load previous state
    state_load "$name" || true

    log_info "Starting supervisor for daemon: $name"
    log_info "Command: $command"

    local restart_count=0
    local consecutive_failures=0

    while [[ $_shutdown_requested -eq 0 ]]; do
        local now=$(date +%s)

        # Launch daemon
        log_info "Launching daemon $name (attempt $((restart_count + 1)))"

        state_set "$name" "status" "starting"
        state_set "$name" "start_time" "$now"

        # Run command in background
        $command &
        local pid=$!
        _current_daemon_pid="$pid"

        state_set "$name" "pid" "$pid"
        state_set "$name" "status" "running"
        pid_write "$name" "$pid"

        log_info "Daemon $name started with PID $pid"

        # Health check loop
        local health_failures=0
        while process_running "$pid" && [[ $_shutdown_requested -eq 0 ]]; do
            sleep "$SUPERVISOR_HEALTH_INTERVAL"

            if ! run_health_check "$name" "$health_check"; then
                ((health_failures++))
                log_warn "Health check failed for $name ($health_failures consecutive failures)"

                if [[ $health_failures -ge $SUPERVISOR_HEALTH_RETRIES ]]; then
                    log_error "Too many health check failures, restarting $name"
                    kill -TERM "$pid" 2>/dev/null || true
                    sleep 2
                    kill -KILL "$pid" 2>/dev/null || true
                    break
                fi
            else
                health_failures=0
                consecutive_failures=0
            fi
        done

        # Wait for process to exit
        wait "$pid" 2>/dev/null || true
        local exit_code=$?

        _current_daemon_pid=""
        pid_remove "$name"

        log_info "Daemon $name exited with code $exit_code"

        # Check if we should restart
        if [[ $_shutdown_requested -eq 1 ]]; then
            break
        fi

        if ! should_restart "$name" "$exit_code"; then
            log_info "Daemon $name will not be restarted"
            state_set "$name" "status" "stopped"
            break
        fi

        # Increment counters
        ((restart_count++))
        ((consecutive_failures++))

        state_set "$name" "restart_count" "$restart_count"
        state_set "$name" "last_restart" "$(date +%s)"
        state_set "$name" "errors" "$(($(state_get "$name" "errors") + 1))"

        # Compute backoff
        local backoff=$(compute_backoff "$consecutive_failures")
        log_info "Waiting ${backoff}s before restart (backoff)"

        state_set "$name" "status" "backoff"
        sleep "$backoff"
    done

    log_info "Supervisor for $name exiting"
}

#===============================================================================
# MULTI-DAEMON COORDINATION
#===============================================================================

declare -A _supervised_daemons

supervisor_register() {
    local name="$1"
    local command="$2"
    local priority="${3:-50}"  # Lower = start first
    local health_check="${4:-health_check_default}"

    _supervised_daemons[$name]="$priority|$command|$health_check"
}

supervisor_start_all() {
    supervisor_init

    # Sort by priority
    local sorted_names=$(
        for name in "${!_supervised_daemons[@]}"; do
            local priority=$(echo "${_supervised_daemons[$name]}" | cut -d'|' -f1)
            echo "$priority|$name"
        done | sort -n | cut -d'|' -f2
    )

    local pids=()

    for name in $sorted_names; do
        local config="${_supervised_daemons[$name]}"
        local command=$(echo "$config" | cut -d'|' -f2)
        local health_check=$(echo "$config" | cut -d'|' -f3)

        log_info "Starting supervised daemon: $name"

        # Run supervisor in background
        (
            supervisor_run "$name" "$command" "$health_check"
        ) &

        local supervisor_pid=$!
        pids+=("$supervisor_pid")

        # Brief delay between starts
        sleep 1
    done

    # Wait for all supervisors
    for pid in "${pids[@]}"; do
        wait "$pid" 2>/dev/null || true
    done
}

supervisor_stop_all() {
    log_info "Stopping all supervised daemons"

    for name in "${!_supervised_daemons[@]}"; do
        local pid=$(pid_read "$name")
        if [[ -n "$pid" ]] && process_running "$pid"; then
            log_info "Stopping $name (PID: $pid)"
            kill -TERM "$pid" 2>/dev/null || true
        fi
    done

    # Wait for graceful shutdown
    sleep "$SUPERVISOR_SHUTDOWN_TIMEOUT"

    # Force kill any remaining
    for name in "${!_supervised_daemons[@]}"; do
        local pid=$(pid_read "$name")
        if [[ -n "$pid" ]] && process_running "$pid"; then
            log_warn "Force killing $name (PID: $pid)"
            kill -KILL "$pid" 2>/dev/null || true
        fi
        pid_remove "$name"
    done
}

#===============================================================================
# STATUS AND MONITORING
#===============================================================================

supervisor_status() {
    echo "=== AGI Daemon Supervisor Status ==="
    echo ""

    for state_file in "$SUPERVISOR_STATE_DIR"/*.state; do
        [[ -f "$state_file" ]] || continue

        source "$state_file"

        local running="NO"
        if [[ -n "$DAEMON_PID" ]] && process_running "$DAEMON_PID"; then
            running="YES"
        fi

        local uptime="N/A"
        if [[ "$DAEMON_START_TIME" -gt 0 ]]; then
            local now=$(date +%s)
            local seconds=$((now - DAEMON_START_TIME))
            uptime=$(printf '%dd %02dh %02dm %02ds' $((seconds/86400)) $((seconds%86400/3600)) $((seconds%3600/60)) $((seconds%60)))
        fi

        printf "%-20s  PID: %-6s  Running: %-3s  Status: %-10s  Restarts: %d  Phi: %.3f  Ticks: %d\n" \
            "$DAEMON_NAME" "${DAEMON_PID:-N/A}" "$running" "$DAEMON_STATUS" \
            "$DAEMON_RESTART_COUNT" "${DAEMON_PHI:-0.5}" "${DAEMON_TICKS:-0}"
        echo "  Uptime: $uptime  Errors: ${DAEMON_ERRORS:-0}"
    done

    echo ""
}

supervisor_metrics() {
    local format="${1:-json}"

    case "$format" in
        json)
            echo "{"
            echo "  \"daemons\": ["
            local first=true
            for state_file in "$SUPERVISOR_STATE_DIR"/*.state; do
                [[ -f "$state_file" ]] || continue
                source "$state_file"

                local running="false"
                [[ -n "$DAEMON_PID" ]] && process_running "$DAEMON_PID" && running="true"

                $first || echo ","
                first=false

                cat <<EOF
    {
      "name": "$DAEMON_NAME",
      "pid": ${DAEMON_PID:-null},
      "running": $running,
      "status": "$DAEMON_STATUS",
      "restart_count": $DAEMON_RESTART_COUNT,
      "phi": ${DAEMON_PHI:-0.5},
      "ticks": ${DAEMON_TICKS:-0},
      "errors": ${DAEMON_ERRORS:-0}
    }
EOF
            done
            echo ""
            echo "  ]"
            echo "}"
            ;;
        prometheus)
            for state_file in "$SUPERVISOR_STATE_DIR"/*.state; do
                [[ -f "$state_file" ]] || continue
                source "$state_file"

                local running=0
                [[ -n "$DAEMON_PID" ]] && process_running "$DAEMON_PID" && running=1

                echo "agi_daemon_running{name=\"$DAEMON_NAME\"} $running"
                echo "agi_daemon_restarts{name=\"$DAEMON_NAME\"} $DAEMON_RESTART_COUNT"
                echo "agi_daemon_phi{name=\"$DAEMON_NAME\"} ${DAEMON_PHI:-0.5}"
                echo "agi_daemon_ticks{name=\"$DAEMON_NAME\"} ${DAEMON_TICKS:-0}"
                echo "agi_daemon_errors{name=\"$DAEMON_NAME\"} ${DAEMON_ERRORS:-0}"
            done
            ;;
    esac
}

#===============================================================================
# DAEMON HEALTH HEARTBEAT (for daemons to call)
#===============================================================================

# Daemons should call this periodically to indicate health
supervisor_heartbeat() {
    local name="$1"
    local phi="${2:-0.5}"
    local ticks="${3:-0}"

    local health_file="$SUPERVISOR_STATE_DIR/${name}.health"
    echo "$(date +%s)|$phi|$ticks" > "$health_file"

    state_set "$name" "phi" "$phi"
    state_set "$name" "ticks" "$ticks"
}

#===============================================================================
# EXPORTS
#===============================================================================

export -f supervisor_init supervisor_run supervisor_register
export -f supervisor_start_all supervisor_stop_all supervisor_status supervisor_metrics
export -f supervisor_heartbeat state_set state_get
export -f log_info log_warn log_error log_debug
export -f health_check_default health_check_tcp health_check_http health_check_file