#!/bin/bash
#                                                                                       #
# SCRIPT: install_primary_monmgo.sh                                                             #
# DATE:   22-May-2017                                                                   #
# REV:    1.0                                                                           #
# DEVELOPED BY: Parablu Systems                                                         #
# DEVELOPER: Vaibhav K
# PURPOSE: Driver Script for Health Check                    #
# PLATFORM: Ubuntu


PARABLU_CONFIG_SCRIPT_PATH=../../config/parablu_config.sh
PARABLU_COMPONENT_SCRIPT_PATH=../../config/parablu_component_config.sh
PARABLU_HEALTH_CHECK_PARACLOUD=./paracloud_health_check.sh
PARABLU_HEALTH_CHECK_BLUKRYPT=./blukrypt_health_check.sh
PARABLU_HEALTH_CHECK_MONGO=./mongo_health_check.sh
PARABLU_HEALTH_CHECK_MSJOB=./msjob_health_check.sh
PARABLU_WATCHDOG_CONFIG=./watchdog_config.sh
PARABLU_WATCHDOG_FUNCTIONS=./watchdog_common_functions.sh

PCB_PRIVACY_GATEWAY="/var/lib/tomcat-pcb1/webapps/BluKryptBuilder/WEB-INF/classes/privacygateway.properties"
source $PARABLU_CONFIG_SCRIPT_PATH
source $PARABLU_HEALTH_CHECK_PARACLOUD
source $PARABLU_HEALTH_CHECK_BLUKRYPT
source $PARABLU_HEALTH_CHECK_MONGO
source $PARABLU_HEALTH_CHECK_MSJOB
source $PARABLU_COMPONENT_SCRIPT_PATH
source $PARABLU_WATCHDOG_CONFIG
source $PARABLU_WATCHDOG_FUNCTIONS

export PATH=$PATH:/sbin
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"

function start_health_check ()
{

        for a in ${SERVER_TYPE[@]};do
                case $a in
                        0) echo "********************************************************************************************************************" >>watchdog.log
        echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_mongo 2>&1 >> health_check_mongo.log
                        # comenting paracloud for time being since same process runs in blukrypt
                        #   monitor_paracloud 2>&1 >> health_check_paracloud.log
                           monitor_blukrypt 2>&1 >> health_check_blukrypt.log
                           echo "********************************************************************************************************************" >>watchdog.log ;;
                        1) monitor_paracloud 2>&1 >> health_check_paracloud.log ;;
                        2) echo "********************************************************************************************************************" >>watchdog.log
        echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_blukrypt 2>&1 >> health_check_blukrypt.log
                           echo "********************************************************************************************************************" >>watchdog.log ;;
                        3)# echo "********************************************************************************************************************" >>watchdog.log
        #echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_mongo 2>&1 >> health_check_mongo.log ;;
         #                  echo "********************************************************************************************************************" >>watchdog.log ;;
			6) echo "********************************************************************************************************************" >>watchdog.log
        echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_msjob 2>&1 >> health_check_msjob.log
                           echo "********************************************************************************************************************" >>watchdog.log ;;

                esac
        done
}

function initialze_cron ()
{
	watchdog_path=`pwd`
	croncmd="cd $watchdog_path && ./health_check.sh"
}

function start_watchdog ()
{

	echo -e "Machine Statistics of `hostname` at `date`" >>watchdog.log
        echo -e "LAST SERVER UPTIME IS:`uptime`" >> watchdog.log

	if [ -f /parablu/mongo_backup_status ]; then
		echo "********************************************************************************************************************" >>watchdog.log
		echo "`cat /parablu/mongo_backup_status`" >>watchdog.log
		echo "********************************************************************************************************************" >>watchdog.log
		echo "" >>watchdog.log
	fi

	service_stop_status
	additional_utilities
	start_health_check
	mkdir -p old_info
	if [ "$BK_ACTIVE" != "false" ]; then
		disk_check
		cpu_memory_stats
		check_mount_points
		check_readonly
	fi


	##### OUTPUTS FROM EXECUTIVE SUMMARY

	if [ -f ../blacklistmongooutput.txt ]; then
                if [ -s ../blacklistmongooutput.txt ]; then

                        echo "File has data"
                        echo "`cat ../blacklistmongooutput.txt`" >>watchdog.log

                else
                        echo "File is empty"

                fi
        fi

        if [ -f ../odbmongooutput.txt ]; then
                if [ -s ../odbmongooutput.txt ]; then

                        echo "File has data"
                        echo "`cat ../odbmongooutput.txt`" >>watchdog.log

                else
                        echo "File is empty"

                fi
        fi

        if [ -f ../o365blacklistmongooutput.txt ]; then
                if [ -s ../o365blacklistmongooutput.txt ]; then

                        echo "File has data"
                        echo "`cat ../o365blacklistmongooutput.txt`" >>watchdog.log

                else
                        echo "File is empty"

                fi
        fi


}

function additional_utilities ()
{
        echo "SERVER VERSION IS: `cat /parablu/installed_version`" >>watchdog.log
	if [ "$BK_ACTIVE" == "false" ]; then

                echo "BLUKRYPT STATUS: DISABLED" >> watchdog.log

        elif [ "$BK_ACTIVE" == "true" ]; then
                echo "BLUKRYPT STATUS: ENABLED" >> watchdog.log

        fi
	if [ "$EXCLUDE_SPEEDTEST" != "true" ]; then
	        curl -s https://raw.githubusercontent.com/sivel/speedtest-cli/master/speedtest.py | python - >>speedtest.log
	        grep Download speedtest.log >>watchdog.log
	        grep Upload speedtest.log  >>watchdog.log
	        sleep 2s
	        rm speedtest.log
	fi
}

function device_exceed_list ()
{
        cur_dir=`pwd`
        cd /parablu/$CLOUD_NAME/upload
        #du -sh * | grep '[0-9]G\>' >device_excced.log
        device_limit='1G'
        du -sh * --threshold=$device_limit >$cur_dir/device_excced.log
                if [ -s $cur_dir/device_excced.log ];
                then
                        echo "********************************************************************************************************************" >> $cur_dir/watchdog.log
                        echo -e "List of Devices more than 1 GB:" >> $cur_dir/watchdog.log
                        echo "`cat $cur_dir/device_excced.log`" >> $cur_dir/watchdog.log
                        echo "********************************************************************************************************************" >> $cur_dir/watchdog.log
                fi
        sleep 2s

        cd $cur_dir
        rm device_excced.log

}

function check_mount_points ()
{
#MOUNT_POINTS
rm $DISK_UNMOUNT_ALERT 2> /dev/null
rm $DISK_UNMOUNT_ALERT_SUBJECT 2> /dev/null

echo "----------------------------------DISK MOUNT/UNMOUNT----------------------------------" >>watchdog.log
for mountp in ${MOUNT_POINTS[@]}
do

        df -h $mountp | grep "$mountp"
        if [ $? -ne 0 ]; then
                #### retry again
                sleep 10s
                df -h $mountp | grep "$mountp"
                if [ $? -ne 0 ]; then
                        echo "Disk $mountp Unmounted">>$DISK_UNMOUNT_ALERT
                        echo "ALERT: DISK UNMOUNT [$CUSTOMER_NAME] Machine `hostname` at `date`" >$DISK_UNMOUNT_ALERT_SUBJECT
			echo "Disk $mountp UNMOUNTED" >>watchdog.log
                fi
        else

                echo "Disk $mountp Mounted"
		echo "Disk $mountp MOUNTED" >>watchdog.log

        fi
done

echo "----------------------------------END OF DISK MOUNT/UNMOUNT----------------------------------" >>watchdog.log

if [ -f $SCRIPT_PATH/$DISK_UNMOUNT_ALERT_SUBJECT ]; then
	echo "">>$DISK_UNMOUNT_ALERT
        df -h >>$DISK_UNMOUNT_ALERT
        sleep 2s
        mount -a
        sleep 3s
        echo "">>$DISK_UNMOUNT_ALERT
        echo "----------------------------------------------------------------------------------" >>$DISK_UNMOUNT_ALERT
        echo "Mounted disks after running mount -a" >>$DISK_UNMOUNT_ALERT
        df -h >>$DISK_UNMOUNT_ALERT
fi

}

function check_readonly ()
{

##Get mail ids
#for mail_id in ${WATCHDOG_MAIL_ADDRESS[@]}
 #               do
  #                      MAILARRAY+=({\"email\":\"$mail_id\"},)
   #     done


    #    tempids=$(echo -e "${MAILARRAY[*]}")
     #   get_mail_id=$(echo $tempids | sed -e 's/.$//')
	get_supportmail


## Check filesystem
echo "---------------------------------- DISK READ ONLY MODE CHECK----------------------------------" >>watchdog.log
for filesys in ${FILE_SYSTEMS[@]}
do

        MSG_SUBJECT="ALERT: DISK $filesys READONLY MODE [$CUSTOMER_NAME] Machine `hostname` at `date`"
        MSG_BODY="Disk $filesys has become read-only"

        touch $filesys/dummy_file && rm -rf $filesys/dummy_file
        if [ $? -eq 0 ]; then
                echo "FileSystem $filesys is Writable"
		echo "FileSystem $filesys is Writable" >>watchdog.log
        else
                echo "FileSystem $filesys is Read Only"
		echo "FileSystem $filesys is Read Only" >> watchdog.log
                curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${MSG_BODY}"'","subjectData":"'"${MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
        fi
done
echo "----------------------------------END OF DISK READONLY MODE----------------------------------" >>watchdog.log
}

function check_pgoverload ()
{
        ##Get mail ids
#for mail_id in ${WATCHDOG_MAIL_ADDRESS[@]}
 #               do
  #                      MAILLARRAY+=({\"email\":\"$mail_id\"},)
   #     done


    #    tempids=$(echo -e "${MAILLARRAY[*]}")
     #   get_mail_id=$(echo $tempids | sed -e 's/.$//')
	get_supportmail

echo 'use parablu001

db.PRIVACY_GATEWAY.find({"gatewayName" : "GATEWAY_NAME"}).forEach(function(e) {print("PGOVERLOADLIMIT=" +e.componentsProperties.pgOverloadLimit)})' >capture_pgoverloadlimit.js

        sed -i "s|GATEWAY_NAME|$BLUKRYPT_IP|g" capture_pgoverloadlimit.js

        sudo mongo $DB1_IP:$MONGODB_PORT -u "neil" -p "parablu" --authenticationDatabase "admin" < capture_pgoverloadlimit.js > capture_pgOverloadLimit

        PGOVERLOADLIMIT=`grep "PGOVERLOADLIMIT" capture_pgOverloadLimit`
        PGOVERLOADLIMIT_VAL=$( echo $PGOVERLOADLIMIT | sed -e 's/PGOVERLOADLIMIT=//')
        MAX_PGLIMIT_LIMIT="50"
        PGLIMIT_MSG_SUBJECT="ALERT: PGOVERLOADLIMIT VALUE $PGOVERLOADLIMIT_VAL [$CUSTOMER_NAME] Machine `hostname` at `date`"
        PGLIMIT_MSG_BODY="PG OVERLOAD LIMIT IS: $PGOVERLOADLIMIT_VAL"
        if [[ ${PGOVERLOADLIMIT_VAL} -lt ${MAX_PGLIMIT_LIMIT} ]];then
                echo "SEND ALERT"
		echo "********************************************************************************************************************" >> watchdog.log
                echo "PG OVERLOAD LIMIT OF $BLUKRYPT_IP IS: $PGOVERLOADLIMIT_VAL" >>watchdog.log
                echo "********************************************************************************************************************" >> watchdog.log
                curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${PGLIMIT_MSG_BODY}"'","subjectData":"'"${PGLIMIT_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
        else
                echo "PG limit is fine"
        fi

}

function check_paracloud_ping ()
{
	count=1
        until [ $count -ge 3 ]
        do

                echo "TRYING TO  PING PARACLOUD...............[`date`]"
                call_paracloudPing=`curl -k /dev/null --max-time 240 --silent --get --write-out '%{http_code}\n'  https://$BLUVAULT_IP/paracloud/ping/`
                echo "$call_paracloudPing" | grep 400
                if [ $? -ne 0 ]; then
                      paracloudpingfail=true
                      echo "PARACLOUD IS NOT PINGING"
                      touch paracloudnotpinging
                else
                      echo "PARACLOUD IS PINGING"
                      rm paracloudnotpinging
                fi



                if [ "$paracloudpingfail" == "true" ]; then
                        apachpid=$(ps -aux | grep -v -e grep -e tail -e less -e vi -e nano | grep apache2 | awk '{print $2}')
                      sortedapachepid=$(echo "$apachpid" | xargs -n1 | sort -u | xargs)
                        sleep 3s
                        service apache2 stop
                        sleep 2s
                        killall -9 apache2
                        sleep 2s
                        if [ ! -z $sortedapachepid ];then
                                echo "Unable to kill PID"
                        fi

                        service apache2 start
                        sleep 10s

                        echo "RETRYING.......[$count] [`date`]"
                else
                        break
                fi
                count=$[$count+1]
                sleep 5s
                paracloudpingfail=''
        done

	if [ -f paracloudnotpinging ]; then
		get_supportmail
        	echo "send alert"
	        APACHKILL_MSG_SUBJECT="ALERT: PARACLOUD PING FAIL!!! [$CUSTOMER_NAME] Machine `hostname` at `date`"
        	APACHKILL_MSG_BODY="Unable to ping Paracloud even after killing apache PID's [ $sortedapachepid ] and Restarting Apache Multiple times."
		echo "****************************************PARACLOUD PING FAIL********************************************************" >> watchdog.log
                echo "Unable to ping Paracloud $BLUVAULT_IP even after killing apache PID's [ $sortedapachepid ] and Restarting Apache Multiple times." >> watchdog.log
                echo "********************************************************************************************************************" >> watchdog.log
	        curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${APACHKILL_MSG_BODY}"'","subjectData":"'"${APACHKILL_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
        	rm paracloudnotpinging
	fi
}

function check_blukrypt_ping ()
{
	count=1
        until [ $count -ge 3 ]
        do

                echo "TRYING TO  PING BLUKRYPT...............[`date`]"
                call_blukryptPing=`curl -k /dev/null --silent --get --write-out '%{http_code}\n'  https://$BLUKRYPT_IP/BluKryptBuilder/ping/`
                echo "$call_blukryptPing" | grep 200
                if [ $? -ne 0 ]; then
                      blukryptpingfail=true
                      echo "BLUKRYPT IS NOT PINGING"
                      touch blukryptnotpinging
                else
                      echo "BLUKRYPT IS PINGING"
                      rm blukryptnotpinging
                fi



                if [ "$blukryptpingfail" == "true" ]; then
                        apachpid=$(ps -aux | grep -v -e grep -e tail -e less -e vi -e nano | grep apache2 | awk '{print $2}')
                      sortedapachepid=$(echo "$apachpid" | xargs -n1 | sort -u | xargs)
                        sleep 3s
                        service apache2 stop
                        sleep 2s
                        killall -9 apache2
                        sleep 2s
                        if [ ! -z $sortedapachepid ];then
                                echo "Unable to kill PID"
                        fi

                        service apache2 start
                        sleep 10s

                        echo "RETRYING.......[$count] [`date`]"
                else
                        break
                fi
                count=$[$count+1]
                sleep 5s
                blukryptpingfail=''
        done

	if [ -f blukryptnotpinging ]; then
		get_supportmail
        	echo "send alert"
	        APACHKILL_MSG_SUBJECT="ALERT: BLUKRYPT PING FAIL!!! [$CUSTOMER_NAME] Machine `hostname` at `date`"
        	APACHKILL_MSG_BODY="Unable to ping Blukrypt even after killing apache PID's [ $sortedapachepid ] and Restarting Apache Multiple times."
		echo "****************************************BLUKRYPT PING FAIL********************************************************" >> watchdog.log
                echo "Unable to ping Blukrypt $BLUKRYPT_IP even after killing apache PID's [ $sortedapachepid ] and Restarting Apache Multiple times." >> watchdog.log
                echo "********************************************************************************************************************" >> watchdog.log
	        curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${APACHKILL_MSG_BODY}"'","subjectData":"'"${APACHKILL_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
        	rm blukryptnotpinging
	fi
}

function service_stop_status ()
{

if [[ "$SERVER_TYPE" == "0" ]] || [[ "$SERVER_TYPE" == "1 2" ]] || [[ "$SERVER_TYPE" == "2" ]];then
        echo 'use parablu001

db.PRIVACY_GATEWAY.find({"gatewayName" : "GATEWAY_NAME"}).forEach(function(e) {print("SERVICESTOP=" +e.isActive)})' >capture_servicestop.js

        sed -i "s|GATEWAY_NAME|$BLUKRYPT_IP|g" capture_servicestop.js

        sudo mongo $DB1_IP:$MONGODB_PORT -u "neil" -p "parablu" --authenticationDatabase "admin" < capture_servicestop.js > capture_servicestop

        SERVICESTOP=`grep "SERVICESTOP" capture_servicestop`
        BK_ACTIVE=$( echo $SERVICESTOP | sed -e 's/SERVICESTOP=//')
else

	###OTHER SERVERS 
	#BK_ACTIVE=true
	echo "NOT BLUKRYPT SERVER"
fi

}

function disk_check ()
{

	## START CHECKING DISK
        for wc_disc in ${WATCHDOG_DISK[@]}
        do
                MEMORY_USED_INSLASH=$(df -h $wc_disc | tail -1 | awk '{print $5}')
                WARN_MEMORY_LIMIT="80%"
                MAX_MEMORY_LIMIT="95%"
                if [[ ${MEMORY_USED_INSLASH%?} -ge ${WARN_MEMORY_LIMIT%?} ]] && [[ ${MEMORY_USED_INSLASH%?} -lt ${MAX_MEMORY_LIMIT%?} ]];then
                        ###### Send Disk Alert
                        Enable_incoming+=("no")
                        df -h $wc_disc | grep "$wc_disc"
                        if [ $? -eq 0 ]; then
                                INITIAL_DISK_STATUS+=([ $wc_disc is ${MEMORY_USED_INSLASH} ])
                        fi

                        if [  -f incoming_stopped ];then
                                echo "Already stopped"
                        else
                                touch INITIAL_ALERT_START
                        fi

                elif [[ ${MEMORY_USED_INSLASH%?} -ge ${MAX_MEMORY_LIMIT%?} ]];then
                        ######Stop All Incoming
                        touch incoming_stopped
                        Enable_incoming+=("no")
                        df -h $wc_disc | grep "$wc_disc"
                        if [ $? -eq 0 ]; then
                                FINAL_DISK_STATUS+=([ $wc_disc is ${MEMORY_USED_INSLASH} ])
                        fi

                elif [[ ${MEMORY_USED_INSLASH%?} -lt ${WARN_MEMORY_LIMIT%?} ]];then
                        Enable_incoming+=("yes")
                        DISK_STATUS+=([ $wc_disc is ${MEMORY_USED_INSLASH} ])

                fi
        done
        ####### END OF DISK CHECK

	##### INITIAL ALERT FLAG CHECK
        if [  -f INITIAL_ALERT_START ];then

                cat alertmailcontent.txt >$INITIAL_ALERT
                sed -i "s|WARN_ALERT|${INITIAL_DISK_STATUS[*]}|" $INITIAL_ALERT
                sed -i "s|MAX_LIMIT|$MAX_MEMORY_LIMIT|" $INITIAL_ALERT
                echo "DISK ALERT in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$INITIAL_ALERT_SUBJECT
                rm INITIAL_ALERT_START

        fi

	#### DISABLING INCOMINING
        DISK_MSG_SUBJECT="DISK ALERT in [$CUSTOMER_NAME] Machine `hostname` at `date`"
        DISK_MSG_BODY="DISK STATUS: ${FINAL_DISK_STATUS[*]}   ${INITIAL_DISK_STATUS[*]} "
        if [  -f incoming_stopped ];then
		get_supportmail
                if [  ! -f last_pgOverloadLimit ]; then
                        if [[ "$SERVER_TYPE" == "3" ]] || [[ "$SERVER_TYPE" == "1" ]] || [[ "$SERVER_TYPE" == "3 1" ]] || [[ "$SERVER_TYPE" == "6" ]];then
                                echo "NO CHANGES MADE.... ONLY TRIGGERING ALERT"
				curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${DISK_MSG_BODY}"'","subjectData":"'"${DISK_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/

                        else
                                cat finalalertmailcontent.txt >$FINAL_ALERT
                                sed -i "s|WARN_LIMIT|$WARN_MEMORY_LIMIT|" $FINAL_ALERT
                                sed -i "s|MAX_ALERT|${FINAL_DISK_STATUS[*]}|" $FINAL_ALERT
                                echo "DISK EXCEEDED in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$FINAL_ALERT_SUBJECT
                                ####Make PG OVERLOAD LIMIT zero                                                         
                                touch last_pgOverloadLimit
                                cp disable_pgoverloadlimit.js disable_pgoverloadlimit_install.js
                                sleep 1s
                                sed -i "s|GATEWAY_NAME|$BLUKRYPT_IP|g" disable_pgoverloadlimit_install.js
                                sleep 2s
                                sudo mongo $DB1_IP:$MONGODB_PORT -u "neil" -p "parablu" --authenticationDatabase "admin" < disable_pgoverloadlimit_install.js > last_pgOverloadLimit
                                PGOVERLOADLIMIT=`grep "PGOVERLOADLIMIT" last_pgOverloadLimit`
                                PGOVERLOADLIMIT_VAL=$( echo $PGOVERLOADLIMIT | sed -e 's/PGOVERLOADLIMIT=//')
                                sed -i "s|PGOVERLOADLIMIT=.*|PGOVERLOADLIMIT="$PGOVERLOADLIMIT_VAL"|" $PARABLU_COMPONENT_SCRIPT_PATH
                                #service tomcat-pcb1 restart
				curl -k -X POST https://$BLUKRYPT_IP/BluKryptBuilder/cloud/$CLOUD_NAME/updateProperties/
                        fi

                else
                        if [[ "$SERVER_TYPE" == "3" ]] || [[ "$SERVER_TYPE" == "1" ]] || [[ "$SERVER_TYPE" == "3 1" ]] || [[ "$SERVER_TYPE" == "6" ]];then
                                echo "DO NOTHING"
                                echo "NO CHANGES MADE.... ONLY TRIGGERING ALERT"
				curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${DISK_MSG_BODY}"'","subjectData":"'"${DISK_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
                        else
                                echo "Incoming already stopped" 
                                cat afterfinalalertmailcontent.txt >$AFTER_FINAL_ALERT
                                sed -i "s|WARN_LIMIT|$WARN_MEMORY_LIMIT|" $AFTER_FINAL_ALERT
                                echo "SERVICE STOPPED in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$AFTER_FINAL_ALERT_SUBJECT
                        fi
		fi
	fi


	### Enabling Incoming
        if [  -f incoming_stopped ];then

                echo "${Enable_incoming[*]}" | grep "no"

                if [ $? -eq 0 ]; then

                        echo "DONT ENABLE INCOMING"

                else
                        rm -rf *.tmp
                        sleep 2s
                        echo "ENABLE INCOMING" 
                        cat restartalert.txt >$RESTART_ALERT
                        echo "SERVICE RESTARTED in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$RESTART_ALERT_SUBJECT

                        if [[ "$SERVER_TYPE" == "3" ]] || [[ "$SERVER_TYPE" == "1" ]] || [[ "$SERVER_TYPE" == "3 1" ]] || [[ "$SERVER_TYPE" == "6" ]];then

                                echo "NO Changes made to Server"
                        else
                                cp enable_pgoverloadlimit.js enable_pgoverloadlimit_install.js
                                sleep 1s
                                sed -i "s|GATEWAY_NAME|$BLUKRYPT_IP|" enable_pgoverloadlimit_install.js
                                sed -i "s|PGOVERLOADLIMIT|$PGOVERLOADLIMIT|" enable_pgoverloadlimit_install.js
                                sudo mongo $DB1_IP:$MONGODB_PORT -u "neil" -p "parablu" --authenticationDatabase "admin" < enable_pgoverloadlimit_install.js
                                #service tomcat-pcb1 restart
				curl -k -X POST https://$BLUKRYPT_IP/BluKryptBuilder/cloud/$CLOUD_NAME/updateProperties/
                                mv last_pgOverloadLimit old_info/last_pgOverloadLimit-$date_time
                        fi

                        mv incoming_stopped old_info/incoming_stopped-$date_time
                fi

        fi	

	if [[ "$SERVER_TYPE" == "0" ]] || [[ "$SERVER_TYPE" == "1 2" ]] || [[ "$SERVER_TYPE" == "2" ]];then
		check_pgoverload
	fi

	if [[ "$SERVER_TYPE" == "0" ]] || [[ "$SERVER_TYPE" == "1 2" ]] || [[ "$SERVER_TYPE" == "1" ]] || [[ "$SERVER_TYPE" == "3 1" ]];then
                check_paracloud_ping
        fi

	if [[ "$SERVER_TYPE" == "0" ]] || [[ "$SERVER_TYPE" == "2" ]];then
                check_blukrypt_ping
        fi

}	

function cpu_memory_stats ()
{
        echo "CPU CORE IS: `cat /proc/cpuinfo | grep "cpu cores"| sed -n 1p | awk -F: '{print$2}'`" >>watchdog.log
        echo "----------------------------------CPU STATS----------------------------------" >>watchdog.log
        cat /proc/cpuinfo | grep -e "processor" -e "vendor_id" -e "cpu family" -e "model" -e "cpu MHz" -e "cpu cores" >>watchdog.log

        echo "----------------------------------END OF CPU STATS----------------------------------" >>watchdog.log

        swap_status=`swapon -show`
        echo "----------------------------------SWAPON STATS----------------------------------" >>watchdog.log
        if [ -z $swap_status ]; then
                echo "SWAPON IS: DISABLED" >>watchdog.log
        else
                echo "SWAPON IS: ENABLED" >>watchdog.log
                swapon -show >>watchdog.log
        fi
        echo "----------------------------------END OF SWAPON----------------------------------" >>watchdog.log

}
rm -rf *.tmp
rm -rf watchdog.log
sleep 1s
start_watchdog

