ES节点的自动重启逻辑：

#!/bin/bash

log="/home/srvop/es_restart.log"

send_mail(){
    to_receiver=""
    subject=$1
    mailbody=$2
    curl -H "Content-type: application/json;charset=utf-8" -X POST  -d '{"to_receiver": $to_receiver, "subject": $subject, "mailbody": $mailbody}'  http://ip:port/login

}


check_es(){
    insName=$1
    port=$2
    ps aux | grep $insName | grep -v grep  > /dev/null  ; proc_status=`echo $?`
    netstat -ntlp | grep $port | grep -v grep > /dev/null ; port_status=`echo $?`
    if [[ $proc_status -ne 0 || $port_status -ne 0  ]]; then
        now=`env LANG=en_US.UTF-8 date +"%Y-%m-%d %H:%M:%S"`
        echo "$now $insName process or $port port non-existent,check again." | tee -a  $log
        sleep 10
        # 第一次判断检测失败后再次检测逻辑
        ps aux | grep $insName | grep -v grep  > /dev/null  ; proc_status=`echo $?`
        netstat -ntlp | grep $port | grep -v grep > /dev/null ; port_status=`echo $?`
        if [[  $proc_status -ne 0 && $port_status -ne 0  ]]; then
            # 第二次仍检测到应用宕机后，启动应用逻辑
            now=`env LANG=en_US.UTF-8 date +"%Y-%m-%d %H:%M:%S"`
            echo "$now $insName process and $port port non-existent,and restart proc now." | tee -a  $log
            cd /srvop/apache/servers/szsc-smt-es-prd-$insName/bin/ ; ./elasticsearch -d
            #cd /srvop/apache/servers/szsc-smt-es-prd-$insName/bin/ ; touch  $insName.txt 
            now=`env LANG=en_US.UTF-8 date +"%Y-%m-%d %H:%M:%S"`
            echo "$now $insName has been restarted." | tee -a  $log
            sleep 120
            # 启动应用后检测是否启动成功逻辑
            ps aux | grep $insName | grep -v grep  > /dev/null  ; proc_status=`echo $?`
            netstat -ntlp | grep $port | grep -v grep > /dev/null ; port_status=`echo $?`
            now=`env LANG=en_US.UTF-8 date +"%Y-%m-%d %H:%M:%S"`
            if [[  $proc_status -eq 0 && $port_status -eq 0 ]]; then
                echo "$now $insName has been restarted successfully." | tee -a  $log
                #send_mail "数据平台ES节点$insName 自动重启成功" "$now  数据平台ES节点$insName 系统检测到宕机，现已重启成功"
            else
                echo "$now $insName has been restarted, but the restart failed. Please check manually." | tee -a  $log
                #send_mail "数据平台ES节点$insName 自动重启失败" "$now  数据平台ES节点$insName 系统检测到宕机，已尝试重启，但重启失败，请人工检查原因."
            fi

        fi


    fi
}

main(){
    check_es ins004 30109 &
    check_es ins005 30110 &
    check_es ins006 30111 &
    wait
}
main