#!/bin/bash
#############################################
#
# date 2013/01/25   create
# update 2013/01/28  Adding duplicate detection
#
# Execution:
# touch /root/sh/mysql_slave_status.sh; chmod 700 /root/sh/mysql_slave_status.sh
# vi /etc/crontab
# 5,15,25,35,45,55 * * * * root /root/sh/mysql_slave_status.sh >> /root/sh/mysql_slave_status.log 2>&1
#############################################
 
######### set variables ############
tmpdir=/tmp
 
#mysqlhost="127.0.0.1"
#mysqlport="3306"
mysqlsocket="/opt/mysql/tmp/mysql.sock"
mysqlbinpath="/opt/mysql/bin"
mysqluser="root"
mysqlpw="NwNRvCb6IMSfQERi"
 
projectname="jiujiu_backup"
mailfromadd="jiujiu@gmail.com"
mailtoadd='allen20170220@gmail.com'
mailccadd='allen20170220@gmail.com'
#mailccadd2='opsteam@service.soonyo.com'
repeat_alarm_time=12           # Repeat alarm time interval ( About *10 minute. value of 12, about two hours)
failure_interval=10            # Interval after a problem is detected, the unit: seconds ( < 25 seconds)
reporting_time=0855            # Must be notified of the time, even if normal.
current_time=$(date +%H%M)     # Current time
 
export LANG=C
export LC_ALL=C
export PATH="$mysqlbinpath":/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 
 
###### do ####################################
echo ""
echo ""
echo "`date` Start monitoring."
 
 
#### function ####
function mailto() {
# mail
/usr/sbin/sendmail -t <<EOF
From: $mailfromadd
To: $mailtoadd
Cc: $mailccadd2
Subject: [replication_check] $projectname slave status ok!
$projectname slave status:
$slaveiostatus
$slavesqlstatus
$slavebehind
EOF
}
 
function mailto2() {
# mail
/usr/sbin/sendmail -t <<EOF
From: $mailfromadd
To: $mailtoadd
Cc: $mailccadd
Subject: [replication_check] $projectname slave status problems!
$projectname slave status:
$slaveiostatus
$lastioerror
$slavesqlstatus
$lastsqlerror
$slavebehind
EOF
}
 
function mailto3() {
# mail
/usr/sbin/sendmail -t <<EOF
From: $mailfromadd
To: $mailtoadd
Cc: $mailccadd
Subject: [replication_check] $projectname slave status problems! (too many times)
$projectname slave status:
$slaveiostatus
$lastioerror
$slavesqlstatus
$lastsqlerror
$slavebehind
EOF
}
 
function mailto4() {
# mail
/usr/sbin/sendmail -t <<EOF
From: $mailfromadd
To: $mailtoadd
Cc: $mailccadd
Subject: [replication_check] $projectname slave status return to normal from the question.
$projectname slave status:
$slaveiostatus
$slavesqlstatus
$slavebehind
EOF
}
 
function showstatus() {
mysql -S $mysqlsocket -u"$mysqluser" -p"$mysqlpw" -e "show slave status\G" > "$tmpdir"/"$projectname"_status.txt
slaveiostatus=`cat "$tmpdir"/"$projectname"_status.txt | grep "Slave_IO_Running" | sed 's/^[ \t]*//g'`
lastioerror=`cat "$tmpdir"/"$projectname"_status.txt | grep "Last_IO_Error" | sed 's/^[ \t]*//g'`
slavesqlstatus=`cat "$tmpdir"/"$projectname"_status.txt | grep "Slave_SQL_Running" | sed 's/^[ \t]*//g'`
lastsqlerror=`cat "$tmpdir"/"$projectname"_status.txt | grep "Last_SQL_Error" | sed 's/^[ \t]*//g'`
slavebehind=`cat "$tmpdir"/"$projectname"_status.txt | grep "Seconds_Behind_Master" | sed 's/^[ \t]*//g'`
slaveiovalue=`cat "$tmpdir"/"$projectname"_status.txt | grep "Slave_IO_Running" | sed 's/^[ \t]*//g' | awk -F ': ' '{print $2}'`
slavesqlvalue=`cat "$tmpdir"/"$projectname"_status.txt | grep "Slave_SQL_Running" | sed 's/^[ \t]*//g' | awk -F ': ' '{print $2}'`
slavebehindvalue=`echo "$slavebehind" | awk -F ': ' '{print $2}' | grep '[0-9]'`
}
 
 
### check ####
test -f "$tmpdir"/"$projectname"_m_value || echo "0" > "$tmpdir"/"$projectname"_m_value
test -f "$tmpdir"/"$projectname"_n_value || echo "0" > "$tmpdir"/"$projectname"_n_value
m=`cat "$tmpdir"/"$projectname"_m_value`
n=`cat "$tmpdir"/"$projectname"_n_value`
 
if   showstatus;sync;sleep 1
     [ "$slaveiovalue" == "Yes" ] && [ "$slavesqlvalue" == "Yes" ] && [ "$slavebehindvalue" -le 3600 ];then
     echo "`date` First detected, $projectname slave status ok!"
     m=0; echo $m > "$tmpdir"/"$projectname"_m_value
elif sleep $failure_interval
     showstatus;sync;sleep 1
     [ "$slaveiovalue" == "Yes" ] && [ "$slavesqlvalue" == "Yes" ] && [ "$slavebehindvalue" -le 3600 ];then
     echo "`date` Second detection, $projectname slave status ok!"
     m=0; echo $m > "$tmpdir"/"$projectname"_m_value
elif sleep $failure_interval
     showstatus;sync;sleep 1
     [ "$slaveiovalue" == "Yes" ] && [ "$slavesqlvalue" == "Yes" ] && [ "$slavebehindvalue" -le 3600 ];then
     echo "`date` Third detection, $projectname slave status ok!"
     m=0; echo $m > "$tmpdir"/"$projectname"_m_value
else
     echo "`date` After three detection, $projectname slave problems!"
     m=$(($m+1)); echo $m > "$tmpdir"/"$projectname"_m_value
fi
 
 
#### log and mail ####
if   [ "$reporting_time" -eq "$current_time" ] && [ "$m" -eq 0 ] && [ "$n" -eq 0 ]; then
     echo "mailto."
     mailto
elif [ "$m" -eq 1 ] && [ "$n" -eq 0 ]; then
     echo "`date` $projectname slave status problems! mailto2."
     echo "$projectname slave status: "
     echo "$slaveiostatus"
     echo "$lastioerror"
     echo "$slavesqlstatus"
     echo "$lastsqlerror"
     echo "$slavebehind"
     mailto2
     n=1; echo $n > "$tmpdir"/"$projectname"_n_value
elif [ "$m" -eq "$repeat_alarm_time" ] && [ "$n" -eq 1 ]; then
     mailto3
     echo "`date` $projectname slave status problems! too many times, mailto3."
     mysql -S $mysqlsocket -u"$mysqluser" -p"$mysqlpw" -e "stop slave"
     mysql -S $mysqlsocket -u"$mysqluser" -p"$mysqlpw" -e "start slave"
     m=0; echo $m > "$tmpdir"/"$projectname"_m_value
elif [ "$m" -eq 0 ] && [ "$n" -eq 1 ]; then
     mailto4
     echo "$projectname slave status return to normal from the question. mailto4."
     n=0; echo $n > "$tmpdir"/"$projectname"_n_value
elif [ "$m" -ne 0 ] && [ "$n" -eq 1 ]; then
     echo "`date` $projectname slave status problems! too many times, nomailto"
fi