Blog

After suffering with a VPS, between load, network problems, and clients complaining I did a dirty fix until the problem got spotted.

#!/bin/bash
######################################################################################
# Derek Demuro, this script is given as is, CopyLEFT                                 #
######################################################################################
######################################################################################
# README                              LEAME                                          #
######################################################################################
# This script is done to attempt reboot on fallen server,
# After reboot message, so if reboot failed, it will send message.
# The reboot only works with local server, as ssh and autonegociation isnt done yet
############################To Set Up#################################################
#
# To set this script up, you'll need to add it to a cronjob to run on boot
# Most linux distros will allow a param <a href="https://twitter.com/reboot" class="twitter-atreply ext">@reboot<span class="ext"><span class="element-invisible"> (link is external)</span></span></a> /(path)/servermon.sh
# So... crontab -e
# At the bottom add: <a href="https://twitter.com/reboot" class="twitter-atreply ext">@reboot<span class="ext"><span class="element-invisible"> (link is external)</span></span></a> /(path)/servermon.sh >> (logname.log)
# REMEMBER TO ADD EXECUTABLE BIT TO THE FILE (777 Permissions)
######################################################################################
# SCRIPT CONFIGURATION                                                               #
######################################################################################
### List of servers to check connection with
readonly SERVERLIST='serverList.txt'
### Where should the log be saved?
readonly LOGNAME='servermonitor.log'
### Who should we mail?
readonly MAILTO='ddemuro@gmail.com'
### How many pings before we call it dead
readonly PINGCOUNT=2
### What is your local ip?
readonly LOCALIP='192.64.81.19'
### How much time between check's?
readonly SLEEPTIME=5
### How many records to keep
readonly CLEARLOGTMS=1000
#Critical load to stop everything and restart
readonly HIGH_LOAD=5
#Wait till load gets to this load
readonly LOW_LOAD=2
#Wait for highload
readonly HIGH=4
#Wait for midload
readonly MID=3
#Wait more for lowload
readonly LOW=1
# Enter the services you wish to start/stop when starting/stopping
readonly SERVICES='apache2 mysql postfix dovecot proftpd'
#####ENABLE SYSTEM RESTART IN CASE MACHINE LOOSES INTERNET?#######
readonly HARD_RESTART_FIX=1
######################################################################################
#################################FUNCTIONS START################################
function checkIfSpecial() {
  if [ $LOCALIP==$1 ]; then
    ###Add special actions here
     
    if [ $HARD_RESTART_FIX -eq 0 ]; then
      shutdown -r now
      mailx -s "Server $line is seems to be down" < /dev/null $MAILTO
    fi
  fi
  return 0
}
 
###Function to START a service
function startService() {
    echo "Starting $1"
    messages=`service $1 start`
    if [ $? -eq "0" ]
    then
        # Service failed to start
        echo $messages
        return 1
    else
        echo "$1 started succesfully"
        return 0
    fi
}
 
###Function to STOP a service
function stopService() {
    echo "Stopping $1"
    messages=`service $1 stop`
    if [ $? -eq "0" ]
    then
        # Service failed to stop
        echo $messages
        return 1
    else
        echo "$1 stopped succesfully"
        return 0
    fi
}
 
#####Check server load, if high stop!
function checkServerLoad() {
    #check = load at the moment
    check=`cat /proc/loadavg | sed 's/\./ /' | awk '{print $1}'`
    #Print of the time the cron started.
    echo "Load at the moment $check"
    #If load > 10
    if [ $check -gt $HIGH_LOAD ]
    then
    ###Shutting down! high load!
    changeServicesStatus 1
        #Moment of shutdown
        date=`date`
        echo 'Highload autorestart did run and load average was: '
        echo 'At: ' $date 'With load: ' $check 'services shutted down successfoully'
        #Sleep while HIGH LOAD
            while [ $check -gt $LOW_LOAD ]
            do
                if [ $check -gt $HIGH ]
                then
                    sleep $HIGH
                else
                    if [ $check -gt $MID ]
                    then
                        sleep $MID
                    else
                        if [ $check -gt $LOW ]
                        then
                            sleep $LOW
                        else
                            sleep 3                     
                        fi                
                    fi
                fi            
                #New check
                check=`cat /proc/loadavg | sed 's/\./ /' | awk '{print $1}'`
            done
        #Continue
        echo 'Restarting on load: ' $check
        ###Start Services
        changeServicesStatus 0
     fi
     return 0
}
 
###Function to clear the log
function changeServicesStatus() {
  if [ $1 -eq 0 ]
  then
    for service in `echo $SERVICES`
    do
      startService $service
    done
  else
    for service in `echo $SERVICES`
    do
      stopService $service
    done
  fi
}
 
###Function to clear the log
function clearLog() {
  echo 'Log cleared' > $LOGNAME
  echo 'Script will run ' $1 ' times then will clear itself'>> $LOGNAME
  return 0
}
 
#################################FUNCTIONS FINISH################################
#################################MAIN SCRIPT FUNC################################
while [ TRUE ]; do
    #add 1 to times
    times=`expr $times + 1`
    ##CLEAR THE LOG
    if [ $times -eq $CLEARLOGTMS ]; then
    clearlog
    $times=0
    fi
     
    checkServerLoad 
     
    ##Run script for every line in serverList
    cat $SERVERLIST | while read line
    do
        # check if there are no blank lines
        if [ ! -z $line ]; then
            ping -c 2 $line > /dev/null
            if [ "$?" -ne 0 ]; then
             
                echo "Something wrong with the server: $line"
             
                checkIfSpecial $line
                # Or do send out mail
            else
                echo "All good: $line"
            fi
        fi
    done
  ##We sleep till new run
  sleep $SLEEPTIME
done

Download “Advanced Service Monitor Script”

Adv_Service_Monitor.sh_.zip – Downloaded 139 times – 2.13 KB
ddemuro
administrator

Sr. Software Engineer with over 10 years of experience. Hobbist photographer and mechanic. Tinkering soul in an endeavor to better understand this world. Love traveling, drinking coffee, and investments.

You may also like

Leave a Reply

This site uses Akismet to reduce spam. Learn how your comment data is processed.

%d bloggers like this: