Flash(1) 4.准备自愈自重启脚本 将以下脚本保存至/home/deepseek/selfhealing.sh shell !/bin/bash 备份本作业的日志(log :主日志 + 每节点步日志) backuptasklogs() { local ts f ts"$(date +%Y%m%d%H%M%S)" mkdir p backup for f in logs/log${SLURMJOBID}; do if [ f "$f" ]; then mv "$f" "backup/$(basename "$f")${ts}" echo "Moved $f > backup/" fi done } 等待服务就绪(进程在就绪前退出则立即返回 1) waitforservice() { echo "Waiting for service to be ready (checking $HEALTHURL)..." while true; do if ! kill 0 "$SERVICEPID" 2>/dev/null; then echo "Service process (PID $SERVICEPID) exited before ready." return 1 fi local HTTPSTATUS HTTPSTATUS"$(curl s o /dev/null w "%{httpcode}" "$HEALTHURL" 2>/dev/null)" if [ "$HTTPSTATUS" "200" ]; then echo "Service is ready (HTTP 200)." return 0 fi echo "Attempt: /health returned $HTTPSTATUS, retrying in ${HEALTHINTERVAL}s..." sleep "${HEALTHINTERVAL}" done } 监控服务健康状态和进程(前台循环,失败返回 1) monitorservice() { echo "Starting service health monitoring (every ${HEALTHINTERVAL}s)..." while true; do sleep "${HEALTHINTERVAL}" if ! kill 0 "$SERVICEPID" 2>/dev/null; then echo "Service process died (PID $SERVICEPID)." return 1 fi local HTTPSTATUS HTTPSTATUS"$(curl s o /dev/null w "%{httpcode}" "$HEALTHURL" 2>/dev/null)" if [ "$HTTPSTATUS" ! "200" ]; then echo "Service unhealthy (HTTP $HTTPSTATUS)." return 1 fi
来自: