Blog

After suffering with a VPS, between load, network problems, and clients complaining I did a dirty fix until the problem got spotted.

#!/bin/bash
######################################################################################
# Derek Demuro, this script is given as is, CopyLEFT                                 #
######################################################################################
######################################################################################
# README                              LEAME                                          #
######################################################################################
# This script is done to attempt reboot on fallen server,
# After reboot message, so if reboot failed, it will send message.
# The reboot only works with local server, as ssh and autonegociation isnt done yet
############################To Set Up#################################################
#
# To set this script up, you'll need to add it to a cronjob to run on boot
# Most linux distros will allow a param <a href="//twitter.com/reboot" class="twitter-atreply ext">@reboot<span class="ext"><span class="element-invisible"> (link is external)</span></span></a> /(path)/servermon.sh
# So... crontab -e
# At the bottom add: <a href="//twitter.com/reboot" class="twitter-atreply ext">@reboot<span class="ext"><span class="element-invisible"> (link is external)</span></span></a> /(path)/servermon.sh >> (logname.log)
# REMEMBER TO ADD EXECUTABLE BIT TO THE FILE (777 Permissions)
######################################################################################
# SCRIPT CONFIGURATION                                                               #
######################################################################################
### List of servers to check connection with
readonly SERVERLIST='serverList.txt'
### Where should the log be saved?
readonly LOGNAME='servermonitor.log'
### Who should we mail?
readonly MAILTO='ddemuro@gmail.com'
### How many pings before we call it dead
readonly PINGCOUNT=2
### What is your local ip?
readonly LOCALIP='192.64.81.19'
### How much time between check's?
readonly SLEEPTIME=5
### How many records to keep
readonly CLEARLOGTMS=1000
#Critical load to stop everything and restart
readonly HIGH_LOAD=5
#Wait till load gets to this load
readonly LOW_LOAD=2
#Wait for highload
readonly HIGH=4
#Wait for midload
readonly MID=3
#Wait more for lowload
readonly LOW=1
# Enter the services you wish to start/stop when starting/stopping
readonly SERVICES='apache2 mysql postfix dovecot proftpd'
#####ENABLE SYSTEM RESTART IN CASE MACHINE LOOSES INTERNET?#######
readonly HARD_RESTART_FIX=1
######################################################################################
#################################FUNCTIONS START################################
function checkIfSpecial() {
  if [ $LOCALIP==$1 ]; then
    ###Add special actions here
     
    if [ $HARD_RESTART_FIX -eq 0 ]; then
      shutdown -r now
      mailx -s "Server $line is seems to be down" < /dev/null $MAILTO
    fi
  fi
  return 0
}
 
###Function to START a service
function startService() {
    echo "Starting $1"
    messages=`service $1 start`
    if [ $? -eq "0" ]
    then
        # Service failed to start
        echo $messages
        return 1
    else
        echo "$1 started succesfully"
        return 0
    fi
}
 
###Function to STOP a service
function stopService() {
    echo "Stopping $1"
    messages=`service $1 stop`
    if [ $? -eq "0" ]
    then
        # Service failed to stop
        echo $messages
        return 1
    else
        echo "$1 stopped succesfully"
        return 0
    fi
}
 
#####Check server load, if high stop!
function checkServerLoad() {
    #check = load at the moment
    check=`cat /proc/loadavg | sed 's/\./ /' | awk '{print $1}'`
    #Print of the time the cron started.
    echo "Load at the moment $check"
    #If load > 10
    if [ $check -gt $HIGH_LOAD ]
    then
    ###Shutting down! high load!
    changeServicesStatus 1
        #Moment of shutdown
        date=`date`
        echo 'Highload autorestart did run and load average was: '
        echo 'At: ' $date 'With load: ' $check 'services shutted down successfoully'
        #Sleep while HIGH LOAD
            while [ $check -gt $LOW_LOAD ]
            do
                if [ $check -gt $HIGH ]
                then
                    sleep $HIGH
                else
                    if [ $check -gt $MID ]
                    then
                        sleep $MID
                    else
                        if [ $check -gt $LOW ]
                        then
                            sleep $LOW
                        else
                            sleep 3                     
                        fi                
                    fi
                fi            
                #New check
                check=`cat /proc/loadavg | sed 's/\./ /' | awk '{print $1}'`
            done
        #Continue
        echo 'Restarting on load: ' $check
        ###Start Services
        changeServicesStatus 0
     fi
     return 0
}
 
###Function to clear the log
function changeServicesStatus() {
  if [ $1 -eq 0 ]
  then
    for service in `echo $SERVICES`
    do
      startService $service
    done
  else
    for service in `echo $SERVICES`
    do
      stopService $service
    done
  fi
}
 
###Function to clear the log
function clearLog() {
  echo 'Log cleared' > $LOGNAME
  echo 'Script will run ' $1 ' times then will clear itself'>> $LOGNAME
  return 0
}
 
#################################FUNCTIONS FINISH################################
#################################MAIN SCRIPT FUNC################################
while [ TRUE ]; do
    #add 1 to times
    times=`expr $times + 1`
    ##CLEAR THE LOG
    if [ $times -eq $CLEARLOGTMS ]; then
    clearlog
    $times=0
    fi
     
    checkServerLoad 
     
    ##Run script for every line in serverList
    cat $SERVERLIST | while read line
    do
        # check if there are no blank lines
        if [ ! -z $line ]; then
            ping -c 2 $line > /dev/null
            if [ "$?" -ne 0 ]; then
             
                echo "Something wrong with the server: $line"
             
                checkIfSpecial $line
                # Or do send out mail
            else
                echo "All good: $line"
            fi
        fi
    done
  ##We sleep till new run
  sleep $SLEEPTIME
done

You may also like

Leave a Reply

Recent Comments