#!/bin/bash
# DEVELOPER: Vaibhav K

PARABLU_COMPONENT_SCRIPT_PATH=../../config/parablu_component_config.sh
PARABLU_CONFIG_SCRIPT_PATH=../../config/parablu_config.sh
PARABLU_WATCHDOG_CONFIG=./watchdog_config.sh
PARABLU_WATCHDOG_FUNCTIONS=./watchdog_common_functions.sh

source $PARABLU_COMPONENT_SCRIPT_PATH
source $PARABLU_CONFIG_SCRIPT_PATH
source $PARABLU_WATCHDOG_FUNCTIONS
source $PARABLU_WATCHDOG_CONFIG

SCRIPT_PATH=`pwd`

export PATH=$PATH:/sbin
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH"

dbport=$(grep port: /etc/mongod.conf)
MONGODB_PORT=$(echo $dbport | sed -e 's/port://' | sed 's/"//g' | tr -d ' ')

function ad_connectivity_check ()
{
call_ad=`curl -o /dev/null --silent --get --write-out '%{http_code}\n'  https://$BLUVAULT_IP/paracloud/versions/2.3/clouds/$CLOUD_NAME/checkAdConnection`

                        echo "$call_ad" | grep 200
                        if [ $? -eq 0 ]; then
                                echo "AD CONNECTIVITY STATUS:" >> ./consolidate/merged.log
                                echo "AD is Connected" >> ./consolidate/merged.log
                                echo "" >> ./consolidate/merged.log
        echo "********************************************************************************************************************" >>./consolidate/merged.log
                        fi

                        echo "$call_ad" | grep 404
                        if [ $? -eq 0 ]; then
                                echo "AD CONNECTIVITY STATUS:" >> ./consolidate/merged.log
                                echo "Unable to connect AD" >> ./consolidate/merged.log
                                echo "" >> ./consolidate/merged.log
        echo "********************************************************************************************************************" >>./consolidate/merged.log
                        fi
}

function odb_connectivity_check ()
{
if [[ "$STORAGE_TYPE" == "3" ]]; then
                        call_odb=`curl -o /dev/null --silent --head --write-out '%{http_code}\n' https://login.windows.net`
                        echo "$call_odb" | grep 200
                        if [ $? -eq 0 ]; then
                                echo "ODB CONNECTIVITY STATUS:" >> ./consolidate/merged.log
                                echo "ODB is Connected" >> ./consolidate/merged.log
                                echo "" >> ./consolidate/merged.log
                                echo "********************************************************************************************************************" >>./consolidate/merged.log
                        else
                                echo "ODB CONNECTIVITY STATUS:" >> ./consolidate/merged.log
                                echo "Unable to connect ODB" >> ./consolidate/merged.log
                                echo "" >> ./consolidate/merged.log
                                echo "********************************************************************************************************************" >>./consolidate/merged.log
                        fi

else
                echo "NOT ODB"
fi

}

function get_cloud_validity ()
{

echo 'use parablu

db.CLOUD.find({}).forEach(function(e){print("E_DATE=" +e.validityPeriod)})' >expirydate.js

sudo mongo localhost:$DBPORT -u "neil" -p "parablu" --authenticationDatabase "admin" < expirydate.js > expiry_date

EXP_DATE=$(grep "E_DATE" expiry_date | sed -e 's/E_DATE=//')

echo "CLOUD NAME: $CLOUD_NAME" >> ./consolidate/merged.log
echo "CLOUD VALIDITY: $EXP_DATE" >> ./consolidate/merged.log
echo "********************************************************************************************************************" >>./consolidate/merged.log

}

function get_ip_list ()
{
if [ "$SERVER_TYPE" == "3 1" ]; then

echo 'use parablu001
db.PRIVACY_GATEWAY.distinct("gatewayName");

db.MSJOB_IP.distinct("MSJOBName");' >iplist.js
else

echo 'use parablu001
db.PRIVACY_GATEWAY.distinct("gatewayName")

db.BLUVAULT.distinct("bluVaultName");

db.MSJOB_IP.distinct("MSJOBName");' >iplist.js
fi

	sudo mongo localhost:$MONGODB_PORT -u "neil" -p "parablu" --authenticationDatabase "admin" < iplist.js >iplist.log

	grep '"' iplist.log | sed -e 's/\[//' | sed -e 's/\]//' | sed -e 's/"//g' | sed -e 's/,//g'  >sorted_iplist.log
	sed -i "/Implicit/d" sorted_iplist.log
	final_iplist=`cat sorted_iplist.log`
	OTHER_WATCHDOG_IP=$(echo "$final_iplist" | xargs -n1 | sort -u | xargs)

}

function pg_check ()
{

##Get mail ids
#for mail_id in ${WATCHDOG_MAIL_ADDRESS[@]}
 #               do
  #                      MAILARRAY+=({\"email\":\"$mail_id\"},)
   #     done


    #    tempids=$(echo -e "${MAILARRAY[*]}")
     #   get_mail_id=$(echo $tempids | sed -e 's/.$//')
	get_supportmail

###########
	echo "PG CHECK STATUS:" >> ./consolidate/merged.log
	echo "" >> ./consolidate/merged.log
	get_ip_list
	for ping_ip in ${OTHER_WATCHDOG_IP[@]}
                do

                        get_host=$(ssh -p $SSH_PORT $SSH_USERNAME@$ping_ip "hostname")
			LAST_THREAD=$(ssh -p $SSH_PORT $SSH_USERNAME@$ping_ip "strings /parablu/log/tomcat-pcb1/BlukryptBuilder.log | grep '##@@' | tail -1")
                        ssh -p $SSH_PORT $SSH_USERNAME@$ping_ip test  -d "/var/lib/tomcat-pcb1/webapps/BluKryptBuilder"

                        if [ $? -eq 0 ]; then

                                call_bkapi=`curl -k /dev/null --max-time 240 --silent --get --write-out '%{http_code}\n'  https://$ping_ip/BluKryptBuilder/ping/`
                                echo "$call_bkapi" | grep 200
                                if [ $? -eq 0 ]; then

                                        echo "PG Machine $ping_ip is UP" >> ./consolidate/merged.log
					echo "LAST THREAD COUNT LINE: $LAST_THREAD" >> ./consolidate/merged.log
                                        echo "" >> ./consolidate/merged.log
                                else
					sleep 1m
					call_bkapi=`curl -k /dev/null --max-time 240 --silent --get --write-out '%{http_code}\n'  https://$ping_ip/BluKryptBuilder/ping/`
					echo "$call_bkapi" | grep 200
					if [ $? -eq 0 ]; then
						echo "PG Machine $ping_ip is UP" >> ./consolidate/merged.log
						echo "LAST THREAD COUNT LINE: $LAST_THREAD" >> ./consolidate/merged.log
	                                        echo "" >> ./consolidate/merged.log
					else
					
	                                        echo "PG Machine $ping_ip is DOWN" >> ./consolidate/merged.log
						echo "LAST THREAD COUNT LINE: $LAST_THREAD" >> ./consolidate/merged.log
	
						PG_MSG_SUBJECT="ALERT: PG Check of [$CUSTOMER_NAME] at `date`"
					        PG_MSG_BODY="PG Machine $ping_ip is DOWN"

                        	                #ssh $SSH_USERNAME@$ping_ip "service apache2 restart"
                                	        #echo "Apache Service restarted" >> pg_ping.log
                                        	echo "" >> ./consolidate/merged.log

						curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${PG_MSG_BODY}"'","subjectData":"'"${PG_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
					fi
                                fi
                        fi

                done
	echo "********************************************************************************************************************" >>./consolidate/merged.log


}

function pbadmin_check ()
{
if [[ "$STORAGE_TYPE" == "3" ]]; then

	##Get mail ids
#	for mail_id in ${WATCHDOG_MAIL_ADDRESS[@]}
 #       do
  #                      MAILARRAY+=({\"email\":\"$mail_id\"},)
   #     done


    #    tempids=$(echo -e "${MAILARRAY[*]}")
     #   get_mail_id=$(echo $tempids | sed -e 's/.$//')
	get_supportmail

###########

	call_pbadmin=`curl -o /dev/null --silent --get --write-out '%{http_code}\n'  https://$BLUVAULT_IP/paracloud/versions/2.4/clouds/$CLOUD_NAME/odb/isPasswordExpired`

        echo "$call_pbadmin" | grep 200
        if [ $? -eq 0 ]; then
        	echo "PB ADMIN PASSWORD STATUS:" >> ./consolidate/merged.log
                echo "PB ADMIN PASSWORD EXPIRED" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
		PBADMIN_MSG_SUBJECT="ALERT: PBADMIN PASSWORD EXPIRED [$CUSTOMER_NAME] at `date`"
		PBADMIN_MSG_BODY="PBADMIN PASSWORD EXPIRED"
		curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${PBADMIN_MSG_BODY}"'","subjectData":"'"${PBADMIN_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
          
	        echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

        echo "$call_pbadmin" | grep 204
        if [ $? -eq 0 ]; then
        	echo "PB ADMIN PASSWORD STATUS:" >> ./consolidate/merged.log
                echo "ODB NOT ENABLED" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
        	echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

	echo "$call_pbadmin" | grep 100
        if [ $? -eq 0 ]; then
                echo "PB ADMIN PASSWORD STATUS:" >> ./consolidate/merged.log
                echo "PB ADMIN PASSWORD NOT EXPIRED" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
        	echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

	echo "$call_pbadmin" | grep 417
        if [ $? -eq 0 ]; then
                echo "PB ADMIN PASSWORD STATUS:" >> ./consolidate/merged.log
                echo "UNABLE TO CONNECT ODB" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
                echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

else
        echo "NOT ODB"
fi
}

function aad_check ()
{
        get_supportmail

###########

	call_aad=`curl -k /dev/null --silent --get --write-out '%{http_code}\n' https://$BLUVAULT_IP/paracloud/versions/2.5/clouds/$CLOUD_NAME/aad/isPasswordExpired`

	echo "$call_aad" | grep 100
        if [ $? -eq 0 ]; then
                echo "AAD ENABLED: YES" >> ./consolidate/merged.log
                echo "AAD Reachable: YES" >> ./consolidate/merged.log
                echo "AAD PASSWORD EXPIRED: NO" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log

                echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

        echo "$call_aad" | grep 204
        if [ $? -eq 0 ]; then
                echo "AAD ENABLED: NO" >> ./consolidate/merged.log
                echo "AAD Reachable: NA" >> ./consolidate/merged.log
                echo "AAD Password Expired: NA" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
                echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

	echo "$call_aad" | grep 200
        if [ $? -eq 0 ]; then
                echo "AAD ENABLED: YES" >> ./consolidate/merged.log
                echo "AAD Reachable: YES" >> ./consolidate/merged.log
                echo "AAD PASSWORD EXPIRED: YES" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
                PBADMIN_MSG_SUBJECT="ALERT: AAD PASSWORD EXPIRED [$CUSTOMER_NAME] at `date`"
                PBADMIN_MSG_BODY="AAD PASSWORD EXPIRED"
                curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${PBADMIN_MSG_BODY}"'","subjectData":"'"${PBADMIN_MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
                echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

        echo "$call_aad" | grep 417
        if [ $? -eq 0 ]; then
                echo "AAD ENABLED: YES" >> ./consolidate/merged.log
                echo "AAD Reachable: NO" >> ./consolidate/merged.log
                echo "AAD PASSWORD EXPIRED: NA" >> ./consolidate/merged.log
                echo "" >> ./consolidate/merged.log
                echo "********************************************************************************************************************" >>./consolidate/merged.log
        fi

}

function secondary_mongo_check ()
{
	echo "SECONDARY MONGO($DB2_IP) STATUS:" >> ./consolidate/merged.log
	ssh -p $SSH_PORT $SSH_USERNAME@$DB2_IP "service mongod status" >> ./consolidate/merged.log
	echo "" >> ./consolidate/merged.log
        echo "********************************************************************************************************************" >>./consolidate/merged.log
}

function ping_statistics ()
{
	echo "MACHINE CONNECTIVITY STATUS:" >> ./consolidate/merged.log
	get_ip_list
	for ping_ip in ${OTHER_WATCHDOG_IP[@]}
                do

			nc -z $ping_ip $SSH_PORT 2> /dev/null
			if [ $? -eq 0 ]; then
				get_host=$(ssh -p $SSH_PORT $SSH_USERNAME@$ping_ip "hostname")
				echo "Machine $get_host is reachable" >> ./consolidate/merged.log
			else 	
				echo "Machine $ping_ip is not reachable" >> ./consolidate/merged.log
			fi

		done
		echo "" >> ./consolidate/merged.log
                echo "********************************************************************************************************************" >>./consolidate/merged.log


}

function single_machine_statistics_collector ()
{

	rm -rf ./consolidate/merged.log
	rm -rf ./consolidate/Backup_OverviewStatus*

	echo "Machine Statistics of [$CUSTOMER_NAME] at $DATE" >$WATCHDOG_SUBJECT
	if [ -f watchdog.log ];then
		ad_connectivity_check
#		odb_connectivity_check
		pbadmin_check
		aad_check
		get_cloud_validity
		cat watchdog.log >> ./consolidate/merged.log
	fi

	rm -rf watchdog.log
	#curl -k -X GET https://$BLUVAULT_IP/paracloud/cloud/$CLOUD_NAME/overview/createExcel/
	sleep 2s
	curl -k -X POST https://$BLUVAULT_IP/paracloud/cloud/$CLOUD_NAME/overview/createExcelForScripts/
	sleep 5s

	if [ -f $BACKUPOVERVIEWREPORT ];then
		cp $BACKUPOVERVIEWREPORT ./consolidate
	fi

	sleep 1s
	chmod 644 ./consolidate/*

	#### CALL MAIL FUNCTION in watchdog_service.sh
	bash watchdog_service.sh

	sleep 10
	rm -rf ./consolidate/merged.log
	rm -rf ./consolidate/Backup_OverviewStatus*
	rm -rf $WATCHDOG_SUBJECT
}

function multi_machine_statistics_collector ()
{


	rm -rf ./consolidate/merged.log
	rm -rf ./consolidate/Backup_OverviewStatus*
	echo "Machine Statistics of [$CUSTOMER_NAME] at $DATE" >$WATCHDOG_SUBJECT
	if [ -f watchdog.log ];then
		ping_statistics
		pg_check
		if [ "$DB1_IP" != "$DB2_IP" ];then
			secondary_mongo_check
		fi
		ad_connectivity_check
#		odb_connectivity_check
		pbadmin_check
		aad_check
		get_cloud_validity
                cat watchdog.log >> ./consolidate/merged.log
		echo "<<============================= END OF MACHINE STATISTICS =============================>>" >> ./consolidate/merged.log
        	echo "" >> ./consolidate/merged.log
	        echo "" >> ./consolidate/merged.log
        fi

	################################### Copying Log files from other machines ########
	
#	counter=1
	get_ip_list
	for remote_ip in ${OTHER_WATCHDOG_IP[@]}
		do

			scp -P $SSH_PORT $SSH_USERNAME@$remote_ip:$SCRIPT_PATH/watchdog.log ./consolidate/
			if [ $? -eq 0 ]; then

				cat ./consolidate/watchdog.log >> ./consolidate/merged.log
#				counter=$((counter+1))
				echo "<<============================= END OF MACHINE STATISTICS =============================>>" >> ./consolidate/merged.log
				echo "" >> ./consolidate/merged.log
				echo "" >> ./consolidate/merged.log
				rm -rf ./consolidate/watchdog.log
				ssh -p $SSH_PORT $SSH_USERNAME@$remote_ip "cd $SCRIPT_PATH && rm -rf watchdog.log"
				sleep 2s
			fi

		done
	####################################################################################
	rm -rf watchdog.log
	sleep 1s
	#curl -k -X GET https://$BLUVAULT_IP/paracloud/cloud/$CLOUD_NAME/overview/createExcel/
	sleep 2s
	curl -k -X POST https://$BLUVAULT_IP/paracloud/cloud/$CLOUD_NAME/overview/createExcelForScripts/
	sleep 5

	if [ "$SERVER_TYPE" == "3" ]
	then
#		scp -P $SSH_PORT $SSH_USERNAME@$BLUVAULT_IP:$BACKUPOVERVIEWREPORT ./consolidate/
		ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cp $BACKUPOVERVIEWREPORT $SCRIPT_PATH/consolidate"
		scp -P $SSH_PORT ./consolidate/merged.log $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate
		scp -P $SSH_PORT $WATCHDOG_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate
		ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "chmod 644 $SCRIPT_PATH/consolidate/*"
		sleep 2s

#		bash watchdog_service.sh
		ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
  	     	sleep 10
		sleep 2s

		###### clear contents of consolidate in this machine
		rm -rf ./consolidate/merged.log
	        rm -rf ./consolidate/Backup_OverviewStatus*
		rm -rf $WATCHDOG_SUBJECT

		###### clear contents of Consolidate in paramail machine
		ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate && rm -rf *"

	else
		######Used for certain clients#######
#		scp $BACKUPOVERVIEWREPORT $SSH_USERNAME@$MAIL_MACHINE:$SCRIPT_PATH/consolidate
		cp $BACKUPOVERVIEWREPORT $SCRIPT_PATH/consolidate

		sleep 1s
		chmod 644 ./consolidate/*
		sleep 1s
		bash watchdog_service.sh
		sleep 10

		rm -rf ./consolidate/merged.log
                rm -rf ./consolidate/Backup_OverviewStatus*
		rm -rf $WATCHDOG_SUBJECT

	fi

}


function multi_machine_disk_collector ()
{
if [ "$SERVER_TYPE" == "3" ]
then

	scp -P $SSH_PORT $INITIAL_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
        scp -P $SSH_PORT $INITIAL_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/

        scp -P $SSH_PORT $FINAL_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
        scp -P $SSH_PORT $FINAL_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/

        scp -P $SSH_PORT $RESTART_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
        scp -P $SSH_PORT $RESTART_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/

        scp -P $SSH_PORT $AFTER_FINAL_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
        scp -P $SSH_PORT $AFTER_FINAL_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
	
	sleep 2s
	ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
	sleep 10
	ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf *tmp"

	sleep 2s
	rm -rf ./consolidate/*tmp
	rm -rf *tmp

else
	
	cp $INITIAL_ALERT ./consolidate/
	cp $INITIAL_ALERT_SUBJECT ./consolidate/
	cp $FINAL_ALERT ./consolidate/
	cp $FINAL_ALERT_SUBJECT ./consolidate/
	cp $RESTART_ALERT ./consolidate/
	cp $RESTART_ALERT_SUBJECT ./consolidate/
	cp $AFTER_FINAL_ALERT ./consolidate/
	cp $AFTER_FINAL_ALERT_SUBJECT ./consolidate/

	sleep 2s
	bash watchdog_service.sh
	sleep 10
	rm -rf ./consolidate/*tmp
	rm -rf *tmp

fi

get_ip_list
for other_ip in ${OTHER_WATCHDOG_IP[@]}
                do
                       
			##########################################################################################################
			scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$INITIAL_ALERT ./consolidate/
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$INITIAL_ALERT_SUBJECT ./consolidate/

			if [ "$SERVER_TYPE" == "3" ]
			then 

				scp -P $SSH_PORT ./consolidate/$INITIAL_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                scp -P $SSH_PORT ./consolidate/$INITIAL_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                sleep 2s
                                ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE test  -f "$SCRIPT_PATH/consolidate/$INITIAL_ALERT_SUBJECT"

                                if [ $? -eq 0 ]; then
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
                                        sleep 15
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf *tmp"
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                        rm -rf ./consolidate/*tmp
                                else
                                        echo "file missing"
                                fi

			else
				if [  -f ./consolidate/$INITIAL_ALERT ]
                       		 then
                                	echo "RUN MAIL SCRIPT"
					bash watchdog_service.sh
					sleep 15
       					rm -rf ./consolidate/*tmp
					ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                       		 fi
			fi
			##########################################################################################################
			scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$FINAL_ALERT ./consolidate/
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$FINAL_ALERT_SUBJECT ./consolidate/

			if [ "$SERVER_TYPE" == "3" ]
                        then

                                scp -P $SSH_PORT ./consolidate/$FINAL_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                scp -P $SSH_PORT ./consolidate/$FINAL_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                sleep 2s
                                ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE test  -f "$SCRIPT_PATH/consolidate/$FINAL_ALERT_SUBJECT"

                                if [ $? -eq 0 ]; then
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
                                        sleep 15
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf *tmp"
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                        rm -rf ./consolidate/*tmp
                                else
                                        echo "file missing"
                                fi

                        else
                                if [  -f ./consolidate/$FINAL_ALERT ]
                                 then
                                        echo "RUN MAIL SCRIPT"
                                        bash watchdog_service.sh
                                        sleep 15
                                        rm -rf ./consolidate/*tmp
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                 fi
                        fi

			##########################################################################################################
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$RESTART_ALERT ./consolidate/
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$RESTART_ALERT_SUBJECT ./consolidate/

			if [ "$SERVER_TYPE" == "3" ]
                        then

                                scp -P $SSH_PORT ./consolidate/$RESTART_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                scp -P $SSH_PORT ./consolidate/$RESTART_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                sleep 2s
                                ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE test  -f "$SCRIPT_PATH/consolidate/$RESTART_ALERT_SUBJECT"

                                if [ $? -eq 0 ]; then
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
                                        sleep 15
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf *tmp"
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                        rm -rf ./consolidate/*tmp
                                else
                                        echo "file missing"
                                fi

                        else
                                if [  -f ./consolidate/$RESTART_ALERT ]
                                 then
                                        echo "RUN MAIL SCRIPT"
                                        bash watchdog_service.sh
                                        sleep 15
                                        rm -rf ./consolidate/*tmp
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                 fi
                        fi

			#########################################################################################################
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$AFTER_FINAL_ALERT ./consolidate/
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$AFTER_FINAL_ALERT_SUBJECT ./consolidate/

			if [ "$SERVER_TYPE" == "3" ]
                        then

                                scp -P $SSH_PORT ./consolidate/$AFTER_FINAL_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                scp -P $SSH_PORT ./consolidate/$AFTER_FINAL_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                sleep 2s
                                ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE test  -f "$SCRIPT_PATH/consolidate/$AFTER_FINAL_ALERT_SUBJECT"

                                if [ $? -eq 0 ]; then
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
                                        sleep 15
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf *tmp"
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                        rm -rf ./consolidate/*tmp
                                else
                                        echo "file missing"
                                fi

                        else
                                if [  -f ./consolidate/$AFTER_FINAL_ALERT ]
                                 then
                                        echo "RUN MAIL SCRIPT"
                                        bash watchdog_service.sh
                                        sleep 15
                                        rm -rf ./consolidate/*tmp
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf *tmp"
                                 fi
                        fi

			######################################################################################################


		done
}

function multi_machine_unmount_disk_collector ()
{

if [ "$SERVER_TYPE" == "3" ]
then

	scp -P $SSH_PORT $DISK_UNMOUNT_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
        scp -P $SSH_PORT $DISK_UNMOUNT_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
	
	sleep 2s
	ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
	sleep 10
	ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf $DISK_UNMOUNT_ALERT"
	ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf $DISK_UNMOUNT_ALERT_SUBJECT"

	sleep 2s
	rm -rf ./consolidate/$DISK_UNMOUNT_ALERT_SUBJECT
	rm -rf ./consolidate/$DISK_UNMOUNT_ALERT
	rm -rf $DISK_UNMOUNT_ALERT_SUBJECT
	rm -rf $DISK_UNMOUNT_ALERT

else
	cp $DISK_UNMOUNT_ALERT_SUBJECT ./consolidate/
	cp $DISK_UNMOUNT_ALERT ./consolidate/
	sleep 2s
	bash watchdog_service.sh
	sleep 10
	rm -rf ./consolidate/$DISK_UNMOUNT_ALERT_SUBJECT
        rm -rf ./consolidate/$DISK_UNMOUNT_ALERT
        rm -rf $DISK_UNMOUNT_ALERT_SUBJECT
        rm -rf $DISK_UNMOUNT_ALERT

fi

get_ip_list
for other_ip in ${OTHER_WATCHDOG_IP[@]}
                do
                       
			##########################################################################################################
			scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$DISK_UNMOUNT_ALERT_SUBJECT ./consolidate/
                        scp -P $SSH_PORT $SSH_USERNAME@$other_ip:$SCRIPT_PATH/$DISK_UNMOUNT_ALERT ./consolidate/

			if [ "$SERVER_TYPE" == "3" ]
			then 
				scp -P $SSH_PORT ./consolidate/$DISK_UNMOUNT_ALERT_SUBJECT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                scp -P $SSH_PORT ./consolidate/$DISK_UNMOUNT_ALERT $SSH_USERNAME@$PARAMAIL_MACHINE:$SCRIPT_PATH/consolidate/
                                sleep 2s
                                ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE test  -f "$SCRIPT_PATH/consolidate/$DISK_UNMOUNT_ALERT_SUBJECT"

                                if [ $? -eq 0 ]; then
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE "cd $SCRIPT_PATH && ./watchdog_service.sh"
                                        sleep 15
                                        ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf $DISK_UNMOUNT_ALERT_SUBJECT"
					ssh -p $SSH_PORT $SSH_USERNAME@$PARAMAIL_MACHINE  "cd $SCRIPT_PATH/consolidate/ && rm -rf $DISK_UNMOUNT_ALERT"
                                        ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf $DISK_UNMOUNT_ALERT_SUBJECT"
					ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf $DISK_UNMOUNT_ALERT"
                                        rm -rf ./consolidate/$DISK_UNMOUNT_ALERT_SUBJECT
					rm -rf ./consolidate/$DISK_UNMOUNT_ALERT
                                else
                                        echo "file missing"
                                fi

			else
				if [  -f ./consolidate/$DISK_UNMOUNT_ALERT ]
                       		 then
                                	echo "RUN MAIL SCRIPT"
					bash watchdog_service.sh
					sleep 15
       					rm -rf ./consolidate/$DISK_UNMOUNT_ALERT_SUBJECT
					rm -rf ./consolidate/$DISK_UNMOUNT_ALERT
					ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf $DISK_UNMOUNT_ALERT_SUBJECT"
					ssh -p $SSH_PORT $SSH_USERNAME@$other_ip  "cd $SCRIPT_PATH/ && rm -rf $DISK_UNMOUNT_ALERT"
                       		 fi
			fi
		done

}

function release_statistics_collector ()
{

if [ "$SERVER_TYPE" == "0" ]
then 
	single_machine_statistics_collector
else
	multi_machine_statistics_collector
	multi_machine_disk_collector
	multi_machine_unmount_disk_collector
fi

}

if [ "$KEY_BASED_SSH" == "true" ]; then
        ssh-agent
        eval $(ssh-agent)
        ssh-add $SSH_KEY_FILE
fi

sleep 2m
release_statistics_collector
if [ "$KEY_BASED_SSH" == "true" ]; then
        killall -9 ssh-agent
fi
