#!/usr/bin/env python3 import ai_brain_env # auto-inserted by update_ai_brain_env_paths.sh """ Watchdog with manager_sync trigger. - On startup: run manager_sync.sh if present & executable. - Periodically: run manager_sync.sh every SYNC_EVERY cycles. - Non-blocking: sync runs in background so watchdog keeps supervising brains. - Logs to ~/Desktop/AI_Brain/logs/watchdog.log and writes manager_sync output to manager_sync.log. """ import os, time, subprocess, signal, sys from pathlib import Path import compileall HOME = Path.home() BRAIN_HOME = HOME / "Desktop" / "AI_Brain" SUB_BRAINS = BRAIN_HOME / "sub_brains" LOGS = BRAIN_HOME / "logs" STATUS = BRAIN_HOME / "status" LOG_FILE = LOGS / "watchdog.log" ACTIONS_FILE = STATUS / "watchdog.actions.log" HEARTBEAT_FILE = STATUS / "watchdog.heartbeat" MANAGER_NOTIFY = STATUS / "manager.notify" MANAGER_SYNC = BRAIN_HOME / "manager_sync.sh" MANAGER_SYNC_LOG = LOGS / "manager_sync.log" CYCLE = 30 SYNC_EVERY = 20 # run manager_sync every N cycles (20 * 30s = 10 minutes) def log(msg): ts = time.strftime("%Y-%m-%d %H:%M:%S") LINE = f"{ts} {msg}" try: with open(LOG_FILE, "a") as f: f.write(LINE + "\n") except Exception: pass print(LINE) def safe_run(cmd): try: subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return True except subprocess.CalledProcessError as e: log(f"[safe_run] cmd failed: {cmd} -> {e}") return False def heal_system(): log("🩺 [WATCHDOG] Healing: chmod +x .sh, normalize python newlines, compileall") try: for p in BRAIN_HOME.rglob("*.sh"): try: p.chmod(p.stat().st_mode | 0o111) except Exception as e: log(f"[heal] chmod failed {p}: {e}") for p in BRAIN_HOME.rglob("*.py"): try: b = p.read_bytes() nb = b.replace(b'\r\n', b'\n').replace(b'\r', b'\n') if nb != b: p.write_bytes(nb) except Exception as e: log(f"[heal] normalize failed {p}: {e}") compileall.compile_dir(str(BRAIN_HOME), force=False, quiet=1) except Exception as e: log(f"[heal] exception: {e}") def pgrep(pattern): try: res = subprocess.run(["pgrep","-f",pattern], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) out = res.stdout.strip() if out: return [int(x) for x in out.split()] except Exception: pass return [] def list_brains(): out = [] if not SUB_BRAINS.exists(): return out for d in sorted(SUB_BRAINS.iterdir()): if d.is_dir(): launch = d / "launch.sh" if launch.exists(): out.append((d.name, launch)) return out def is_running_by_launch(launch_path): pids = pgrep(str(launch_path)) return len(pids) > 0 def restart_service(name, launch_path): log(f"🚀 [WATCHDOG] Restarting {name} (launch {launch_path})") try: subprocess.run(f"pkill -f \"{str(SUB_BRAINS/name)}\"", shell=True) except Exception: pass try: subprocess.Popen([str(launch_path), "--bg"]) time.sleep(0.5) pids = pgrep(str(launch_path)) with open(ACTIONS_FILE, "a") as f: f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} restarted {name}\n") with open(MANAGER_NOTIFY, "a") as f: f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} watchdog restarted {name}\n") log(f"[WATCHDOG] restart triggered for {name}, pids={pids}") except Exception as e: log(f"[WATCHDOG] restart failed for {name}: {e}") shutdown = False def handle_sigterm(signum, frame): global shutdown log(f"[WATCHDOG] received signal {signum}, shutting down") shutdown = True def write_heartbeat(): try: HEARTBEAT_FILE.write_text(time.strftime("%Y-%m-%dT%H:%M:%SZ")) except Exception as e: log(f"[WATCHDOG] heartbeat write failed: {e}") def run_manager_sync_background(): """If manager_sync.sh exists and is executable, run it in background and log output.""" try: if MANAGER_SYNC.exists() and os.access(MANAGER_SYNC, os.X_OK): log("[WATCHDOG] launching manager_sync.sh in background") # spawn manager_sync and redirect stdout/stderr to manager_sync.log with open(MANAGER_SYNC_LOG, "ab") as out: proc = subprocess.Popen([str(MANAGER_SYNC)], stdout=out, stderr=out) # record action for manager with open(ACTIONS_FILE, "a") as f: f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} triggered manager_sync pid={proc.pid}\n") with open(MANAGER_NOTIFY, "a") as f: f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} watchdog triggered manager_sync pid={proc.pid}\n") log(f"[WATCHDOG] manager_sync started pid={proc.pid}") return True else: log("[WATCHDOG] manager_sync missing or not executable; skipping sync") return False except Exception as e: log(f"[WATCHDOG] failed to start manager_sync: {e}") return False def main_loop(): global shutdown signal.signal(signal.SIGTERM, handle_sigterm) signal.signal(signal.SIGINT, handle_sigterm) log("🐶 [WATCHDOG] started (with manager_sync support)") # initial heal + initial sync heal_system() sync_counter = 0 try: run_manager_sync_background() except Exception as e: log(f"[WATCHDOG] initial sync failed: {e}") while not shutdown: try: heal_system() brains = list_brains() for name, launch in brains: try: if not is_running_by_launch(launch): log(f"[WATCHDOG] {name} not running — attempting restart") restart_service(name, launch) except Exception as e: log(f"[WATCHDOG] error checking {name}: {e}") write_heartbeat() # sync scheduling sync_counter += 1 if sync_counter >= SYNC_EVERY: sync_counter = 0 try: run_manager_sync_background() except Exception as e: log(f"[WATCHDOG] periodic sync failed: {e}") except Exception as e: log(f"[WATCHDOG] main loop exception: {e}") for _ in range(int(CYCLE/1)): if shutdown: break time.sleep(1) log("🐶 [WATCHDOG] exiting — stopping discovered brains") for name, _ in list_brains(): try: subprocess.run(f"pkill -f \"{str(SUB_BRAINS/name)}\"", shell=True) except Exception: pass log("🐶 [WATCHDOG] exit complete") if __name__ == '__main__': main_loop()