process_manager.py

import subprocess
import os
import time
import signal
import sys
from multiprocessing import Process
from monitoring.health_check import start_health_server

# Global flag for graceful shutdown
shutdown_flag = False

def signal_handler(signum, frame):
    """Handle shutdown signals"""
    global shutdown_flag
    print(f"Received signal {signum}. Initiating graceful shutdown...")
    shutdown_flag = True

def start_process(command):
    """Starts a process and keeps it alive"""
    global shutdown_flag
    restart_count = 0
    
    while not shutdown_flag:
        try:
            print(f"Starting command: {command} (attempt {restart_count + 1})")
            process = subprocess.Popen(command, shell=True)
            process.wait()
            
            if shutdown_flag:
                break
                
            restart_count += 1
            print(f"Process '{command}' exited. Restart count: {restart_count}")
            
            # Add delay between restarts to prevent rapid cycling
            if restart_count > 5:
                print(f"Too many restarts for '{command}'. Waiting 30 seconds...")
                time.sleep(30)
            else:
                time.sleep(5)
                
        except KeyboardInterrupt:
            print(f"KeyboardInterrupt received for command: {command}")
            break
        except Exception as e:
            print(f"Unexpected error for command '{command}': {e}")
            time.sleep(10)

def run():
    """Main process manager"""
    # Set up signal handlers
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)
    
    processes = [
        Process(target=start_process, args=("rq worker warmup",)),
        Process(target=start_process, args=("rq worker warmup_reply",)),
        Process(target=start_process, args=("rq worker warmup_read",)),
        Process(target=start_health_server)
    ]
    
    try:
        for p in processes:
            p.start()
            print(f"Started process: {p.name}")
        
        # Wait for processes with proper cleanup
        while any(p.is_alive() for p in processes):
            time.sleep(1)
            if shutdown_flag:
                break
    
    except KeyboardInterrupt:
        print("Main process interrupted. Shutting down...")
    
    finally:
        # Clean shutdown
        print("Terminating all processes...")
        for p in processes:
            if p.is_alive():
                p.terminate()
                p.join(timeout=5)
                if p.is_alive():
                    print(f"Force killing process: {p.name}")
                    p.kill()

if __name__ == "__main__":
    run()