Janady13/missionmatch / ops/m4/missionmatch-recover.sh

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/missionmatch / ops/m4/missionmatch-recover.sh
Date	Not specified in source
Repository	Janady13/missionmatch
Revision	720d47fae476b0d38477ae4401697d372b1ea2bc
Branch	feature/opportunity-to-outcome
Path or archive member	ops/m4/missionmatch-recover.sh
Original SHA-256	6100814ce91b177a5ca952967246d1e87bf52e375b8066d8d69f9ae0f25c79fd
Git blob SHA	7ed1998c8ef6b6eac7aee83fd2de6b8b9d889faa
Reading-copy SHA-256	6100814ce91b177a5ca952967246d1e87bf52e375b8066d8d69f9ae0f25c79fd

Original source ↗

Every recorded occurrence

These 3 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/missionmatch / ops/m4/missionmatch-recover.sh	feature/opportunity-to-outcome	720d47fae476b0d38477ae4401697d372b1ea2bc	Open original
Janady13/missionmatch / ops/m4/missionmatch-recover.sh	master	1e9249840a4302ec82e123b9c9cb6f75badea3e4	Open original
Janady13/missionmatch / ops/m4/missionmatch-recover.sh	feature/aioe-upgrade	2178e7298d0e5d9af83afa12402fd587f0ba6ed8	Open original

Recovered source transcript

Download complete source text ↓112 lines · 4.5 KB

#!/bin/bash
# missionMATCH M4 service recovery. Idempotently ensures the two services this node hosts are running,
# then verifies both. Safe to run repeatedly: it starts a service only when that service is down, so it is
# both the one command manual fix over SSH and the periodic self heal the LaunchAgent calls at boot.
#
#   1. PostgreSQL 16 (the missionMATCH backing store), PGDATA on the external MEGAMIND volume.
#   2. The MEGAMIND inference gateway on 8090 (proxies the local Ollama runtime for embeddings and generation).
#
# It writes a timestamped line per action so the launchd log tells the recovery story after a reboot.

set -u
PATH=/opt/homebrew/bin:/opt/homebrew/opt/postgresql@16/bin:/usr/bin:/bin:/usr/sbin:/sbin
export PATH

PGBIN=/opt/homebrew/opt/postgresql@16/bin
PGDATA=/Volumes/MEGAMIND/missionmatch-pg/data
PGLOG=/Volumes/MEGAMIND/missionmatch-pg/server.log
GW="$HOME/megamind-gateway"
GWLOG="$HOME/megamind-gateway.log"
OLLAMA="$HOME/ollama/ollama"
OLLAMALOG="$HOME/ollama-serve.log"
GENMODEL=llama3.1:8b

log() { echo "$(date '+%Y-%m-%d %H:%M:%S') recover: $*"; }

# genOK probes a REAL generation (not just the port), because after a reboot the Ollama server can be up while
# its model runner is stuck with zero weights loaded and returns an empty response. A non empty response field
# means generation is healthy. The timeout is generous so a legitimate cold model load is not mistaken for a
# failure.
genOK() {
  curl -s -m 60 http://127.0.0.1:11434/api/generate \
    -d "{\"model\":\"$GENMODEL\",\"prompt\":\"ok\",\"stream\":false}" 2>/dev/null \
    | grep -q '"response":"[^"]'
}

# ---------- 1. PostgreSQL ----------
if "$PGBIN/pg_isready" -q -h /tmp -p 5432 2>/dev/null; then
  log "postgres already up"
else
  log "postgres down, starting (PGDATA=$PGDATA)"
  nohup "$PGBIN/postgres" -D "$PGDATA" > "$PGLOG" 2>&1 </dev/null &
  for _ in $(seq 1 20); do
    "$PGBIN/pg_isready" -q -h /tmp -p 5432 2>/dev/null && break
    sleep 1
  done
  if "$PGBIN/pg_isready" -q -h /tmp -p 5432 2>/dev/null; then
    log "postgres up"
  else
    log "postgres FAILED to start. Check $PGLOG. If the log is empty, grant Full Disk Access to $PGBIN/postgres (TCC blocks the external volume otherwise)."
  fi
fi

# ---------- 2. Ollama model runtime (the gateway proxies it for generation) ----------
# The gateway depends on Ollama, so heal Ollama first. A reboot can leave the server up but the model runner
# stuck, so probe a real generation, not just the port. If the server is not running, start it; if it is
# running but generation returns empty (a stuck runner), restart it cleanly.
if ! pgrep -f "ollama serve" >/dev/null 2>&1; then
  if [ -x "$OLLAMA" ]; then
    log "ollama serve not running, starting it"
    nohup "$OLLAMA" serve > "$OLLAMALOG" 2>&1 </dev/null &
    sleep 6
  else
    log "ollama binary $OLLAMA missing, cannot start it"
  fi
fi
if genOK; then
  log "ollama generation healthy"
elif [ -x "$OLLAMA" ]; then
  log "ollama generation not producing text (stuck runner), restarting ollama serve"
  pkill -f "llama-server" 2>/dev/null
  pkill -f "ollama serve" 2>/dev/null
  sleep 3
  nohup "$OLLAMA" serve > "$OLLAMALOG" 2>&1 </dev/null &
  for _ in $(seq 1 20); do
    genOK && break
    sleep 3
  done
  genOK && log "ollama generation restored" || log "ollama generation still down, check $OLLAMALOG"
fi

# ---------- 3. MEGAMIND gateway (needs Ollama on 11434) ----------
if curl -fsS -m 5 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
  log "gateway already up"
elif [ ! -x "$GW" ]; then
  log "gateway binary $GW missing or not executable, cannot start it"
else
  log "gateway down, starting on 0.0.0.0:8090"
  MEGAMIND_ADDR=0.0.0.0:8090 nohup "$GW" > "$GWLOG" 2>&1 </dev/null &
  for _ in $(seq 1 12); do
    curl -fsS -m 5 http://127.0.0.1:8090/healthz >/dev/null 2>&1 && break
    sleep 1
  done
  if curl -fsS -m 5 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
    log "gateway up"
  else
    log "gateway FAILED to start. Check $GWLOG. Confirm Ollama is up: curl http://127.0.0.1:11434/api/tags"
  fi
fi

# ---------- 4. Verify ----------
PG=$("$PGBIN/psql" -U postgres -d missionmatch -tAc 'select 1' 2>/dev/null | tr -d '[:space:]')
GWH=$(curl -s -o /dev/null -w '%{http_code}' -m 5 http://127.0.0.1:8090/healthz 2>/dev/null)
GEN=down
genOK && GEN=up
log "verify: postgres_query=${PG:-down} gateway_healthz=${GWH:-000} ollama_generation=${GEN}"
if [ "$PG" = "1" ] && [ "$GWH" = "200" ] && [ "$GEN" = "up" ]; then
  log "all services healthy"
  exit 0
fi
log "one or more services are NOT healthy, see the lines above"
exit 1