Janady13/ai-brain-core / sub_brains/watchdog/watchdog.py

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/ai-brain-core / sub_brains/watchdog/watchdog.py
Date	Not specified in source
Repository	Janady13/ai-brain-core
Revision	f7275582f13ada80bfb3b1008089800ca26f4a23
Branch	main
Path or archive member	sub_brains/watchdog/watchdog.py
Original SHA-256	819b3423082a7a331c26e713353e70ca109eba8dc2a0b477ec146599026e34f2
Git blob SHA	3fdb62381937adab1cc27b75fcf2127cbad27c15
Reading-copy SHA-256	819b3423082a7a331c26e713353e70ca109eba8dc2a0b477ec146599026e34f2

Original source ↗

Every recorded occurrence

These 3 records contain identical original source bytes. Their paths and revisions remain part of the archive.

Original record	Branch or archive	Revision / identifier	Original location
Janady13/ai-brain-core / sub_brains/watchdog/watchdog.py	main	f7275582f13ada80bfb3b1008089800ca26f4a23	Open original
Janady13/ai-brain-core / sub_brains/watchdog/watchdog.py	claude/agi-architecture-diagram-01GJ7CohfKvLb77yiKpHqd6d	365850e27004318edd582881225024026b118c2d	Open original
Janady13/ai-brain-core / sub_brains/watchdog/watchdog.py	healer-core	0fe4cbbaa91912149f1a8e32205371ff67017268	Open original

Recovered source transcript

Download complete source text ↓194 lines · 6.8 KB

#!/usr/bin/env python3
import ai_brain_env  # auto-inserted by update_ai_brain_env_paths.sh
"""
Watchdog with manager_sync trigger.

 - On startup: run manager_sync.sh if present & executable.
 - Periodically: run manager_sync.sh every SYNC_EVERY cycles.
 - Non-blocking: sync runs in background so watchdog keeps supervising brains.
 - Logs to ~/Desktop/AI_Brain/logs/watchdog.log and writes manager_sync output to manager_sync.log.
"""
import os, time, subprocess, signal, sys
from pathlib import Path
import compileall

HOME = Path.home()
BRAIN_HOME = HOME / "Desktop" / "AI_Brain"
SUB_BRAINS = BRAIN_HOME / "sub_brains"
LOGS = BRAIN_HOME / "logs"
STATUS = BRAIN_HOME / "status"

LOG_FILE = LOGS / "watchdog.log"
ACTIONS_FILE = STATUS / "watchdog.actions.log"
HEARTBEAT_FILE = STATUS / "watchdog.heartbeat"
MANAGER_NOTIFY = STATUS / "manager.notify"
MANAGER_SYNC = BRAIN_HOME / "manager_sync.sh"
MANAGER_SYNC_LOG = LOGS / "manager_sync.log"

CYCLE = 30
SYNC_EVERY = 20  # run manager_sync every N cycles (20 * 30s = 10 minutes)

def log(msg):
    ts = time.strftime("%Y-%m-%d %H:%M:%S")
    LINE = f"{ts} {msg}"
    try:
        with open(LOG_FILE, "a") as f:
            f.write(LINE + "\n")
    except Exception:
        pass
    print(LINE)

def safe_run(cmd):
    try:
        subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        return True
    except subprocess.CalledProcessError as e:
        log(f"[safe_run] cmd failed: {cmd} -> {e}")
        return False

def heal_system():
    log("🩺 [WATCHDOG] Healing: chmod +x .sh, normalize python newlines, compileall")
    try:
        for p in BRAIN_HOME.rglob("*.sh"):
            try:
                p.chmod(p.stat().st_mode | 0o111)
            except Exception as e:
                log(f"[heal] chmod failed {p}: {e}")
        for p in BRAIN_HOME.rglob("*.py"):
            try:
                b = p.read_bytes()
                nb = b.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
                if nb != b:
                    p.write_bytes(nb)
            except Exception as e:
                log(f"[heal] normalize failed {p}: {e}")
        compileall.compile_dir(str(BRAIN_HOME), force=False, quiet=1)
    except Exception as e:
        log(f"[heal] exception: {e}")

def pgrep(pattern):
    try:
        res = subprocess.run(["pgrep","-f",pattern], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
        out = res.stdout.strip()
        if out:
            return [int(x) for x in out.split()]
    except Exception:
        pass
    return []

def list_brains():
    out = []
    if not SUB_BRAINS.exists():
        return out
    for d in sorted(SUB_BRAINS.iterdir()):
        if d.is_dir():
            launch = d / "launch.sh"
            if launch.exists():
                out.append((d.name, launch))
    return out

def is_running_by_launch(launch_path):
    pids = pgrep(str(launch_path))
    return len(pids) > 0

def restart_service(name, launch_path):
    log(f"🚀 [WATCHDOG] Restarting {name} (launch {launch_path})")
    try:
        subprocess.run(f"pkill -f \"{str(SUB_BRAINS/name)}\"", shell=True)
    except Exception:
        pass
    try:
        subprocess.Popen([str(launch_path), "--bg"])
        time.sleep(0.5)
        pids = pgrep(str(launch_path))
        with open(ACTIONS_FILE, "a") as f:
            f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} restarted {name}\n")
        with open(MANAGER_NOTIFY, "a") as f:
            f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} watchdog restarted {name}\n")
        log(f"[WATCHDOG] restart triggered for {name}, pids={pids}")
    except Exception as e:
        log(f"[WATCHDOG] restart failed for {name}: {e}")

shutdown = False
def handle_sigterm(signum, frame):
    global shutdown
    log(f"[WATCHDOG] received signal {signum}, shutting down")
    shutdown = True

def write_heartbeat():
    try:
        HEARTBEAT_FILE.write_text(time.strftime("%Y-%m-%dT%H:%M:%SZ"))
    except Exception as e:
        log(f"[WATCHDOG] heartbeat write failed: {e}")

def run_manager_sync_background():
    """If manager_sync.sh exists and is executable, run it in background and log output."""
    try:
        if MANAGER_SYNC.exists() and os.access(MANAGER_SYNC, os.X_OK):
            log("[WATCHDOG] launching manager_sync.sh in background")
            # spawn manager_sync and redirect stdout/stderr to manager_sync.log
            with open(MANAGER_SYNC_LOG, "ab") as out:
                proc = subprocess.Popen([str(MANAGER_SYNC)], stdout=out, stderr=out)
            # record action for manager
            with open(ACTIONS_FILE, "a") as f:
                f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} triggered manager_sync pid={proc.pid}\n")
            with open(MANAGER_NOTIFY, "a") as f:
                f.write(f"{time.strftime('%Y-%m-%d %H:%M:%S')} watchdog triggered manager_sync pid={proc.pid}\n")
            log(f"[WATCHDOG] manager_sync started pid={proc.pid}")
            return True
        else:
            log("[WATCHDOG] manager_sync missing or not executable; skipping sync")
            return False
    except Exception as e:
        log(f"[WATCHDOG] failed to start manager_sync: {e}")
        return False

def main_loop():
    global shutdown
    signal.signal(signal.SIGTERM, handle_sigterm)
    signal.signal(signal.SIGINT, handle_sigterm)
    log("🐶 [WATCHDOG] started (with manager_sync support)")
    # initial heal + initial sync
    heal_system()
    sync_counter = 0
    try:
        run_manager_sync_background()
    except Exception as e:
        log(f"[WATCHDOG] initial sync failed: {e}")
    while not shutdown:
        try:
            heal_system()
            brains = list_brains()
            for name, launch in brains:
                try:
                    if not is_running_by_launch(launch):
                        log(f"[WATCHDOG] {name} not running — attempting restart")
                        restart_service(name, launch)
                except Exception as e:
                    log(f"[WATCHDOG] error checking {name}: {e}")
            write_heartbeat()
            # sync scheduling
            sync_counter += 1
            if sync_counter >= SYNC_EVERY:
                sync_counter = 0
                try:
                    run_manager_sync_background()
                except Exception as e:
                    log(f"[WATCHDOG] periodic sync failed: {e}")
        except Exception as e:
            log(f"[WATCHDOG] main loop exception: {e}")
        for _ in range(int(CYCLE/1)):
            if shutdown:
                break
            time.sleep(1)
    log("🐶 [WATCHDOG] exiting — stopping discovered brains")
    for name, _ in list_brains():
        try:
            subprocess.run(f"pkill -f \"{str(SUB_BRAINS/name)}\"", shell=True)
        except Exception:
            pass
    log("🐶 [WATCHDOG] exit complete")

if __name__ == '__main__':
    main_loop()