#!/bin/bash
#                                                                                       #
# SCRIPT: install_primary_monmgo.sh                                                             #
# DATE:   22-May-2017                                                                   #
# REV:    1.0                                                                           #
# DEVELOPED BY: Parablu Systems                                                         #
# PURPOSE: Driver Script for Health Check                    #
# PLATFORM: Ubuntu


PARABLU_CONFIG_SCRIPT_PATH=../../config/parablu_config.sh
PARABLU_COMPONENT_SCRIPT_PATH=../../config/parablu_component_config.sh
PARABLU_HEALTH_CHECK_PARACLOUD=./paracloud_health_check.sh
PARABLU_HEALTH_CHECK_BLUKRYPT=./blukrypt_health_check.sh
PARABLU_HEALTH_CHECK_MONGO=./mongo_health_check.sh
PARABLU_WATCHDOG_CONFIG=./watchdog_config.sh
PCB_PRIVACY_GATEWAY="/var/lib/tomcat-pcb1/webapps/BluKryptBuilder/WEB-INF/classes/privacygateway.properties"
source $PARABLU_CONFIG_SCRIPT_PATH
source $PARABLU_HEALTH_CHECK_PARACLOUD
source $PARABLU_HEALTH_CHECK_BLUKRYPT
source $PARABLU_HEALTH_CHECK_MONGO
source $PARABLU_COMPONENT_SCRIPT_PATH
source $PARABLU_WATCHDOG_CONFIG

export PATH=$PATH:/sbin

function start_health_check ()
{

        for a in ${SERVER_TYPE[@]};do
                case $a in
                        0) echo "********************************************************************************************************************" >>watchdog.log
        echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_mongo 2>&1 >> health_check_mongo.log
                        # comenting paracloud for time being since same process runs in blukrypt
                        #   monitor_paracloud 2>&1 >> health_check_paracloud.log
                           monitor_blukrypt 2>&1 >> health_check_blukrypt.log
                           echo "********************************************************************************************************************" >>watchdog.log ;;
                        1) monitor_paracloud 2>&1 >> health_check_paracloud.log ;;
                        2) echo "********************************************************************************************************************" >>watchdog.log
        echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_blukrypt 2>&1 >> health_check_blukrypt.log
                           echo "********************************************************************************************************************" >>watchdog.log ;;
                        3) echo "********************************************************************************************************************" >>watchdog.log
        echo -e "PROCESS_NAME\t | PROCESS_STATUS\t | PID\t\t | PROCESS_CPU_PERCENTAGE\t | PROCESS_MEMORY_PERCENTAGE\t" >>watchdog.log
                           monitor_mongo 2>&1 >> health_check_mongo.log
                           echo "********************************************************************************************************************" >>watchdog.log ;;

                esac
        done
}

function initialze_cron ()
{
	watchdog_path=`pwd`
	croncmd="cd $watchdog_path && ./health_check.sh"
}

function start_watchdog ()
{
MEMORY_USED_INSLASH=$(df -h /parablu | tail -1 | awk '{print $5}')
WARN_MEMORY_LIMIT="80%"
MAX_MEMORY_LIMIT="95%"

		mkdir -p old_info
		if [[ ${MEMORY_USED_INSLASH%?} -ge ${WARN_MEMORY_LIMIT%?} ]] && [[ ${MEMORY_USED_INSLASH%?} -lt ${MAX_MEMORY_LIMIT%?} ]];then
                	###### Send Disk Alert
	                echo -e "Machine Statistics of `hostname` at `date`" >>watchdog.log
			echo -e "LAST SERVER UPTIME IS:`uptime`" >> watchdog.log
			check_mount_points
			additional_utilities
#                        device_exceed_list

			if [ -f /parablu/mongo_backup_status ]; then
			        echo "********************************************************************************************************************" >>watchdog.log
			        echo "`cat /parablu/mongo_backup_status`" >>watchdog.log
			        echo "********************************************************************************************************************" >>watchdog.log
			        echo "" >>watchdog.log
			fi
			start_health_check
			sleep 2

			if [  -f incoming_stopped ];then
				cat afterfinalalertmailcontent.txt >$AFTER_FINAL_ALERT
				sed -i "s|WARN_LIMIT|$WARN_MEMORY_LIMIT|" $AFTER_FINAL_ALERT
				echo "SERVICE STOPPED in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$AFTER_FINAL_ALERT_SUBJECT
			else
				cat alertmailcontent.txt >$INITIAL_ALERT
				sed -i "s|WARN_ALERT|$MEMORY_USED_INSLASH|" $INITIAL_ALERT
                                sed -i "s|MAX_LIMIT|$MAX_MEMORY_LIMIT|" $INITIAL_ALERT
				echo "DISK ALERT in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$INITIAL_ALERT_SUBJECT
                	fi

			if [ -f mongo_stopped ]; then
				service mongod stop
			fi

			if [ -f tomcat_pcb1_stopped ]; then
                                service tomcat-pcb1 stop
                        fi



			initialze_cron
			cronjob="*/30 * * * * $croncmd"
		        ( crontab -l | grep -v -F "$croncmd" ; echo "$cronjob" ) | crontab -

			check_readonly
                

        	elif [[ ${MEMORY_USED_INSLASH%?} -ge ${MAX_MEMORY_LIMIT%?} ]];then
                	######Stop All Incoming and send alert
			echo -e "Machine Statistics of `hostname` at `date`" >>watchdog.log
                        echo -e "LAST SERVER UPTIME IS:`uptime`" >> watchdog.log
			check_mount_points
			additional_utilities
 #                       device_exceed_list

			if [ -f /parablu/mongo_backup_status ]; then
			        echo "********************************************************************************************************************" >>watchdog.log
			        echo "`cat /parablu/mongo_backup_status`" >>watchdog.log
			        echo "********************************************************************************************************************" >>watchdog.log
			        echo "" >>watchdog.log
			fi
                        start_health_check
                        sleep 2

	                cat finalalertmailcontent.txt >$FINAL_ALERT
			sed -i "s|WARN_LIMIT|$WARN_MEMORY_LIMIT|" $FINAL_ALERT
                        sed -i "s|MAX_ALERT|$MEMORY_USED_INSLASH|" $FINAL_ALERT
			echo "DISK EXCEEDED in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$FINAL_ALERT_SUBJECT

			if [  ! -f last_pgOverloadLimit ]; then
				
				if [[ "$SERVER_TYPE" == "3" ]];then

					service mongod stop
					touch mongo_stopped

				elif [[ "$SERVER_TYPE" == "1" ]]; then

					service tomcat-pcb1 stop
					touch tomcat_pcb1_stopped

				else
					source $PCB_PRIVACY_GATEWAY
					echo $pgOverloadLimit >last_pgOverloadLimit
	
					sed -i "/pgOverloadLimit/d" $PCB_PRIVACY_GATEWAY
					echo "pgOverloadLimit=0" >> $PCB_PRIVACY_GATEWAY
					service tomcat-pcb1 restart
				fi
			fi

			touch incoming_stopped

			initialze_cron
	                cronjob="*/30 * * * * $croncmd"
        	        ( crontab -l | grep -v -F "$croncmd" ; echo "$cronjob" ) | crontab -

			check_readonly


	        elif [[ ${MEMORY_USED_INSLASH%?} -lt ${WARN_MEMORY_LIMIT%?} ]];then
			echo -e "Machine Statistics of `hostname` at `date`" >>watchdog.log
			echo -e "LAST SERVER UPTIME IS:`uptime`" >> watchdog.log
			check_mount_points
			additional_utilities
  #                      device_exceed_list
			if [ -f /parablu/mongo_backup_status ]; then
			        echo "********************************************************************************************************************" >>watchdog.log
			        echo "`cat /parablu/mongo_backup_status`" >>watchdog.log
			        echo "********************************************************************************************************************" >>watchdog.log
			        echo "" >>watchdog.log
			fi
	                start_health_check              
			sleep 2
	
			if [  -f incoming_stopped ];then

				cat restartalert.txt >$RESTART_ALERT
				echo "SERVICE RESTARTED in [$CUSTOMER_NAME] Machine `hostname` at `date`" >$RESTART_ALERT_SUBJECT

				if [[ "$SERVER_TYPE" == "3" ]];then

					service mongod start
					mv mongo_stopped old_info/mongo_stopped-$date_time


				elif [[ "$SERVER_TYPE" == "1" ]]; then

					service tomcat-pcb1 start
					mv tomcat_pcb1_stopped old_info/tomcat_pcb1_stopped-$date_time

				else
					sed -i "/pgOverloadLimit/d" $PCB_PRIVACY_GATEWAY
					echo "pgOverloadLimit=`cat last_pgOverloadLimit`" >> $PCB_PRIVACY_GATEWAY
					service tomcat-pcb1 restart
					mv last_pgOverloadLimit old_info/last_pgOverloadLimit-$date_time
				fi

				mv incoming_stopped old_info/incoming_stopped-$date_time
			fi

			initialze_cron
                	cronjob="*/30 * * * * $croncmd"
	                ( crontab -l | grep -v -F "$croncmd" ; echo "$cronjob" ) | crontab -

			check_readonly
	
        	fi
}

function additional_utilities ()
{
        echo "SERVER VERSION IS: `cat /parablu/installed_version`" >>watchdog.log
	if [ "$EXCLUDE_SPEEDTEST" != "true" ]; then
	        curl -s https://raw.githubusercontent.com/sivel/speedtest-cli/master/speedtest.py | python - >>speedtest.log
	        grep Download speedtest.log >>watchdog.log
	        grep Upload speedtest.log  >>watchdog.log
	        sleep 2s
	        rm speedtest.log
	fi
}

function device_exceed_list ()
{
        cur_dir=`pwd`
        cd /parablu/$CLOUD_NAME/upload
        #du -sh * | grep '[0-9]G\>' >device_excced.log
        device_limit='1G'
        du -sh * --threshold=$device_limit >$cur_dir/device_excced.log
                if [ -s $cur_dir/device_excced.log ];
                then
                        echo "********************************************************************************************************************" >> $cur_dir/watchdog.log
                        echo -e "List of Devices more than 1 GB:" >> $cur_dir/watchdog.log
                        echo "`cat $cur_dir/device_excced.log`" >> $cur_dir/watchdog.log
                        echo "********************************************************************************************************************" >> $cur_dir/watchdog.log
                fi
        sleep 2s

        cd $cur_dir
        rm device_excced.log

}

function check_mount_points ()
{
#MOUNT_POINTS
rm $DISK_UNMOUNT_ALERT
rm $DISK_UNMOUNT_ALERT_SUBJECT

for mountp in ${MOUNT_POINTS[@]}
do

        df -h $mountp | grep "$mountp"
        if [ $? -ne 0 ]; then
                #### retry again
                sleep 10s
                df -h $mountp | grep "$mountp"
                if [ $? -ne 0 ]; then
                        echo "Disk $mountp Unmounted">>$DISK_UNMOUNT_ALERT
                        echo "ALERT: DISK UNMOUNT [$CUSTOMER_NAME] Machine `hostname` at `date`" >$DISK_UNMOUNT_ALERT_SUBJECT
                fi
        else

                echo "Disk $mountp Mounted"

        fi
done

if [ -f $SCRIPT_PATH/$DISK_UNMOUNT_ALERT_SUBJECT ]; then
	echo "">>$DISK_UNMOUNT_ALERT
        df -h >>$DISK_UNMOUNT_ALERT
        sleep 2s
        mount -a
        sleep 3s
        echo "">>$DISK_UNMOUNT_ALERT
        echo "----------------------------------------------------------------------------------" >>$DISK_UNMOUNT_ALERT
        echo "Mounted disks after running mount -a" >>$DISK_UNMOUNT_ALERT
        df -h >>$DISK_UNMOUNT_ALERT
fi

}

function check_readonly ()
{

##Get mail ids
for mail_id in ${WATCHDOG_MAIL_ADDRESS[@]}
                do
                        MAILARRAY+=({\"email\":\"$mail_id\"},)
        done


        tempids=$(echo -e "${MAILARRAY[*]}")
        get_mail_id=$(echo $tempids | sed -e 's/.$//')


## Check filesystem
for filesys in ${FILE_SYSTEMS[@]}
do

        MSG_SUBJECT="ALERT: DISK $filesys READONLY MODE [$CUSTOMER_NAME] Machine `hostname` at `date`"
        MSG_BODY="Disk $filesys has become read-only"

        touch $filesys/dummy_file && rm -rf $filesys/dummy_file
        if [ $? -eq 0 ]; then
                echo "FileSystem $filesys is Writable"
        else
                echo "FileSystem $filesys is Read Only"
                curl -k -X PUT -H "Content-Type: application/json" -d '{"bodyData":"'"${MSG_BODY}"'","subjectData":"'"${MSG_SUBJECT}"'","recipents":['"${get_mail_id}"']}' https://$PARAMAIL_MACHINE/paramail/send/customMail/
        fi
done

}


rm -rf *.tmp
rm -rf watchdog.log
sleep 1s
start_watchdog

