546 lines
16 KiB
Bash
Executable File
546 lines
16 KiB
Bash
Executable File
#!/bin/ksh
|
|
#
|
|
# SCRIPT: rsync_daily_copy.ksh
|
|
# AUTHOR: Randy Michael
|
|
# DATE: 11/10/2007
|
|
# REV: 3.2.Prod
|
|
#
|
|
# PURPOSE: This script is used to replicate Oracle .dbf
|
|
# files between the "master" DB server and the two
|
|
# OLTP Oracle servers. The copy method used is
|
|
# rsync. For this method to work, the Oracle DBA must
|
|
# first put the database tables that reside in the
|
|
# copy filesystems into READ-ONLY mode. Before starting
|
|
# rsync copy sessions, this script searches for a file,
|
|
# defined by the $READYTORUN_FILE variable, that is
|
|
# placed on the system by the Oracle DBA team; when
|
|
# the file is found this script will execute
|
|
# all 36 rsync sessions, 18 to each OLTP server,
|
|
# at the same time, then waits for all sessions to
|
|
# complete, both locally and on the remote servers.
|
|
# After the rsync copy sessions complete the copy is
|
|
# verified by matching file sizes between the master
|
|
# copy file and the target copy files. When verified
|
|
# successful this script writes a file to the system,
|
|
# defined by the $COMPLETE_FILE variable, to signal
|
|
# the Oracle DBAs to put the DB back into READ-WRITE
|
|
# mode, copy the metadata over, build the tables and
|
|
# attach the DB on the OLTP side servers. Should a
|
|
# failure occur a file, defined by the $RSYNCFAILED_FILE
|
|
# variable, is written to the system to signal to the
|
|
# Oracle DBA team an rsync copy process failure.
|
|
#
|
|
#
|
|
# EXIT CODES:
|
|
# 0 ==> Normal execution.
|
|
# 1 ==> The value assigned to DAY is not
|
|
# an integer 1 or 2.
|
|
# 2 ==> Remote host is not pingable.
|
|
# 3 ==> Copy verification failed. One or
|
|
# more local files are not the same
|
|
# size as the remote files.
|
|
# 4 ==> No machines are defined to copy
|
|
# data to.
|
|
# 5 ==> Exited on a trapped signal.
|
|
#
|
|
# set -x # Uncomment to debug this script
|
|
#
|
|
# set -n # Uncomment to check the script's syntax
|
|
# # without any execution. Do not forget to
|
|
# # recomment this line!
|
|
#
|
|
########################################
|
|
# REVISION LIST:
|
|
########################################
|
|
#
|
|
# IF YOU MODIFY THIS SCRIPT DOCUMENT THE CHANGE(S)!!!
|
|
#
|
|
########################################
|
|
#
|
|
# Revised by: Randy Michael
|
|
# Revision Date: 7/23/2007
|
|
# Revision: Changed the script to process
|
|
# all 36 mount points at a time.
|
|
#
|
|
########################################
|
|
#
|
|
# Revised by:
|
|
# Revision Date:
|
|
# Revision:
|
|
#
|
|
##############################################
|
|
# DEFINE FILES AND GLOBAL VARIABLES HERE
|
|
##############################################
|
|
|
|
typeset -i DAY
|
|
EMAIL_FROM=data_support@gamma
|
|
export PATH=$PATH:/usr/local/bin
|
|
WORK_DIR=/usr/local/bin
|
|
LOGFILE=${WORK_DIR}/rsync_daily_copy.log
|
|
SEARCH_DIR=/orabin/apps/oracle/dbadm/general/bin
|
|
READYTORUN_FILE=${SEARCH_DIR}/readytocopy.txt
|
|
COMPLETE_FILE=${SEARCH_DIR}/copycomplete.txt
|
|
RSYNCFAILED_FILE=${SEARCH_DIR}/copyfailed.txt
|
|
MAILMESSAGEFILE=${WORK_DIR}/email_message.out
|
|
THIS_SCRIPT=$(basename $0)
|
|
BG_PID_LIST=
|
|
TOTAL=0
|
|
THIS_HOST=$(hostname)
|
|
[[ $THIS_HOST = gamma ]] && MACHINE_LIST="alpha-rsync bravo-rsync"
|
|
[[ $THIS_HOST = gamma-dg ]] && MACHINE_LIST="alpha-rsync bravo-rsync"
|
|
|
|
# Setup the correct echo command usage. Many Linux
|
|
# distributions will execute in BASH even if the
|
|
# script specifies Korn shell. BASH shell requires
|
|
# we use echo -e when we use \n, \c, etc.
|
|
|
|
case $SHELL in
|
|
*/bin/bash) alias echo="echo -e"
|
|
;;
|
|
esac
|
|
|
|
##############################################
|
|
# DEFINE FUNCTIONS HERE
|
|
##############################################
|
|
|
|
usage ()
|
|
{
|
|
echo "\nUSAGE: $THIS_SCRIPT Day"
|
|
echo "\nWhere Day is 1 or 2\n"
|
|
}
|
|
|
|
##############################################
|
|
|
|
cleanup_exit ()
|
|
{
|
|
# If this script is executing, then something failed!
|
|
|
|
[ $1 ] && EXIT_CODE=$1
|
|
|
|
echo "\n$THIS_SCRIPT is exiting on non-zero exit code: $EXIT_CODE"
|
|
echo "\nPerforming cleanup..."
|
|
echo "Removing $READYTORUN_FILE"
|
|
rm -f $READYTORUN_FILE >/dev/null 2>&1
|
|
echo "Removing $COMPLETE_FILE"
|
|
rm -f $COMPLETE_FILE >/dev/null 2>&1
|
|
echo "\nCreating $RSYNCFAILED_FILE"
|
|
echo "\nRsync failed on $THIS_HOST with exit code $EXIT_CODE $(date)\n"\
|
|
| tee -a $RSYNCFAILED_FILE
|
|
echo "\nCleanup Complete...Exiting..."
|
|
return $EXIT_CODE
|
|
exit $EXIT_CODE
|
|
}
|
|
|
|
##############################################
|
|
|
|
trap_exit ()
|
|
{
|
|
echo "\nERROR: EXITING ON A TRAPPED SIGNAL!\n"
|
|
echo "\nRSYNC ERROR: $THIS_HOST -- Rsync process exited abnormally on a \
|
|
trapped signal $(date)!" > $MAILMESSAGEFILE
|
|
|
|
mailx -r "$EMAIL_FROM" -s "SYNC ERROR: $THIS_HOST -- Rsync process exited \
|
|
abnormally on a trapped signal!" data_support < $MAILMESSAGEFILE
|
|
|
|
sleep 2 # Allow the email to go out
|
|
cleanup_exit 5
|
|
return 5
|
|
exit 5
|
|
}
|
|
|
|
##############################################
|
|
|
|
verify_copy ()
|
|
{
|
|
#set -x
|
|
|
|
MYLOGFILE=${WORK_DIR}/verify_rsync_copy_day${DAY}.log
|
|
>$MYLOGFILE
|
|
ERROR=0
|
|
|
|
# Enclose this loop so we can redirect output to the log file
|
|
# with one assignment at the bottom of the function
|
|
{
|
|
# Put a header for the verification log file
|
|
|
|
echo "\nRsync copy verification between $THIS_HOST and machines $MACHINE_LIST\n\n" >$MYLOGFILE
|
|
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
for LOC_MP in $(df | grep oradata_dm_[0-2][0-9] | awk '{print $7}')
|
|
do
|
|
LS_FILES=$(find $LOC_MP -type f)
|
|
for FL in $LS_FILES
|
|
do
|
|
LOC_FS=$(ls -l $FL | awk '{print $5}' 2>&1)
|
|
|
|
# This sed statement changes the "m" to $DAY
|
|
REM_FL=$(echo $FL | sed s/oradata_dm_/oradata_d${DAY}_/g)
|
|
|
|
REM_FS=$(rsh $M ls -l $REM_FL | awk '{print $5}' 2>&1)
|
|
echo "Checking File: $FL"
|
|
echo "Local $THIS_HOST size:\t$LOC_FS"
|
|
echo "Checking Remote File: $REM_FL"
|
|
echo "$M size:\t$REM_FS"
|
|
if [ "$LOC_FS" -ne "$REM_FS" ]
|
|
then
|
|
echo "ERROR: File size mismatch between $THIS_HOST and $M"
|
|
echo "File is: $FL"
|
|
ERROR=1
|
|
fi
|
|
done
|
|
done
|
|
done
|
|
|
|
if (( ERROR != 0 ))
|
|
then
|
|
# Record the failure in the log file
|
|
|
|
echo "\n\nRSYNC ERROR: $THIS_HOST Rsync copy failed...file size \
|
|
mismatch...\n\n" | tee -a $MYLOGFILE
|
|
|
|
# Send email notification with file size log
|
|
|
|
mailx -r "$EMAIL_FROM" -s "RSYNC ERROR: $THIS_HOST Rsync copy failed\
|
|
...file size mismatch -- log attached" data_support < $MYLOGFILE
|
|
|
|
echo "\nERROR: Rsync copy Failed!"
|
|
echo "\n\nCheck log file: $MYLOGFILE\n\n"
|
|
echo "\n...Exiting...\n"
|
|
cleanup_exit 3
|
|
return 3
|
|
else
|
|
echo "\nSUCCESS: Rsync copy completed successfully..."
|
|
echo "\nAll file sizes match...\n"
|
|
fi
|
|
} | tee -a $MYLOGFILE
|
|
|
|
}
|
|
|
|
########################################
|
|
|
|
ready_to_run ()
|
|
{
|
|
# set -x
|
|
# This function looks for a file on the system
|
|
# defined by the $READYTORUN_FILE variable. The
|
|
# presents presence of this file indicates we are ready
|
|
# to run this script. This file will contain a
|
|
# number 1 or 2 identifying the day we are
|
|
# working with.
|
|
|
|
if [ -r ${READYTORUN_FILE} ]
|
|
then
|
|
cat ${READYTORUN_FILE}
|
|
else
|
|
echo "NOT_READY"
|
|
fi
|
|
}
|
|
|
|
##############################################
|
|
|
|
elapsed_time ()
|
|
{
|
|
SEC=$1
|
|
(( SEC < 60 )) && echo "[Elapsed time: $SEC seconds]\c"
|
|
|
|
(( SEC >= 60 && SEC < 3600 )) && echo "[Elapsed time: $(( SEC / 60 )) \
|
|
min $(( SEC % 60 )) sec]\c"
|
|
|
|
(( SEC > 3600 )) && echo "[Elapsed time: $(( SEC / 3600 )) hr $(( (SEC % 3600) / 60 )) \
|
|
min $(( (SEC % 3600) % 60 )) sec]\c"
|
|
}
|
|
|
|
##############################################
|
|
# BEGINNING OF MAIN
|
|
##############################################
|
|
|
|
# Set a trap
|
|
|
|
trap 'trap_exit' 1 2 3 5 6 11 14 15 17
|
|
|
|
# Save the old log file
|
|
|
|
cp -f $LOGFILE ${LOGFILE}.yesterday \
|
|
>$LOGFILE
|
|
|
|
# Enclose the entire main part of the script in
|
|
# curly braces so we can redirect all output of
|
|
# the shell script with a single redirection
|
|
# at the bottom of the script to the $LOGFILE
|
|
|
|
{
|
|
echo "\n[[ $THIS_SCRIPT started execution $(date) ]]\n"
|
|
|
|
# Ensure that target machines are defined
|
|
|
|
if [ -z "$MACHINE_LIST" ]
|
|
then
|
|
echo "\nERROR: No machines are defined to copy data to..."
|
|
echo "\nRSYNC ERROR: $THIS_HOST has no machines are defined \
|
|
to copy data to..." > $MAILMESSAGEFILE
|
|
mailx -r "$EMAIL_FROM" -s "RSYNC ERROR: $THIS_HOST has no machines \
|
|
defined to copy data to" data_support < $MAILMESSAGEFILE
|
|
echo "...Unable to continue...Exiting..."
|
|
|
|
cleanup_exit 4
|
|
exit 4
|
|
fi
|
|
|
|
# Checking for currently running versions of this script
|
|
|
|
echo "Checking for Currently Runnning Versions of this Script"
|
|
|
|
MYPID=$$ # Capture this scripts PID
|
|
|
|
MYOTHERPROCESSES=$(ps -ef | grep $THIS_SCRIPT | grep -v $MYPID \
|
|
| grep -v grep | awk '{print $2}')
|
|
|
|
if [[ "$MYOTHERPROCESSES" != "" ]]
|
|
then
|
|
echo "\WARNING: Another version of this script is running...Killing it!"
|
|
echo "Killing the following processe(es)...$MYOTHERPROCESSES"
|
|
kill -9 $MYOTHERPROCESSES
|
|
echo "\nNOTICE: Sleeping for 1 minute to allow both local
|
|
and remote rsync sessions to terminate, if they exist...\n"
|
|
sleep 60 # Allow any rsync sessions to stop both locally and remotely
|
|
else
|
|
echo "No other versions running...proceeding"
|
|
fi
|
|
|
|
# Remove the file that indicates the rsync copy
|
|
# is mounted and ready to use, if it exists.
|
|
|
|
rm -f $COMPLETE_FILE >/dev/null 2>&1
|
|
|
|
# Remove the file that indicates the rsync copy
|
|
# failed and exited on a non-zero exit code, if
|
|
# it exists.
|
|
|
|
rm -f $RSYNCFAILED_FILE >/dev/null 2>&1
|
|
|
|
|
|
# Search for the file indicating we are ready to proceed
|
|
|
|
# Send notification
|
|
|
|
echo "\nDaily Rsync Copy Process Began on Host $THIS_HOST $(date)" \
|
|
> $MAILMESSAGEFILE
|
|
echo "\nStarted looking for $READYTORUN_FILE" >> $MAILMESSAGEFILE
|
|
|
|
mailx -r "$EMAIL_FROM" -s "Rsync Copy Process Began Looking for Startup \
|
|
File on $THIS_HOST" data_support < $MAILMESSAGEFILE
|
|
|
|
echo "\nSearching for the file: ${READYTORUN_FILE}"
|
|
RUN_STATUS=$(ready_to_run)
|
|
until [[ $RUN_STATUS != "NOT_READY" ]]
|
|
do
|
|
date
|
|
echo "$READYTORUN_FILE is not present...Sleeping 5 minutes..."
|
|
sleep 300
|
|
RUN_STATUS=$(ready_to_run)
|
|
done
|
|
date
|
|
echo "Found file: $READYTORUN_FILE -- Ready to proceed..."
|
|
DAY=$RUN_STATUS
|
|
|
|
|
|
# Test the value assigned to the DAY variable
|
|
|
|
echo "Testing variable assignment for the DAY variable"
|
|
|
|
if (( DAY != 1 && DAY != 2 ))
|
|
then
|
|
echo "\nRSYNC ERROR: $THIS_HOST -- The value assigned to \
|
|
the DAY variable" > $MAILMESSAGEFILE
|
|
echo "==> $DAY - is not an integer 1 or 2...Exiting..."
|
|
>> $MAILMESSAGEFILE
|
|
mailx -r "$EMAIL_FROM" -s "ERROR: $THIS_HOST -- Value assigned \
|
|
to the DAY variable $DAY is invalid" data_support < $MAILMESSAGEFILE
|
|
|
|
usage
|
|
cleanup_exit
|
|
exit 1
|
|
fi
|
|
|
|
# Ensure the remote machines are reachable through
|
|
# the network by sending 1 ping.
|
|
|
|
echo "Verifying the target machines are pingable...\n"
|
|
for M in $MACHINE_LIST
|
|
do
|
|
echo "Pinging $M..."
|
|
ping -c1 $M >/dev/null 2>&1
|
|
if (( $? != 0 ))
|
|
then
|
|
echo "RSYNC ERROR: $M is not pingable from $THIS_HOST" \
|
|
> $MAILMESSAGEFILE
|
|
echo "The rsync copy process cannot continue...Exiting" \
|
|
>> $MAILMESSAGEFILE
|
|
mailx -r "$EMAIL_FROM" -s "RSYNC ERROR: $M is not pingable
|
|
from $THIS_HOST" data_support < $MAILMESSAGEFILE
|
|
|
|
echo "\nERROR: $M host is not pingable...cannot continue..."
|
|
echo "...EXITING...\n"
|
|
sleep 2
|
|
cleanup_exit 2
|
|
exit 2
|
|
else
|
|
echo "Pinging $M succeeded..."
|
|
fi
|
|
done
|
|
|
|
# Notify start of rsync processing by email
|
|
|
|
echo "Rsync Copy Process for Day $DAY Starting Execution on \
|
|
$THIS_HOST $(date)" > $MAILMESSAGEFILE
|
|
|
|
mailx -r "$EMAIL_FROM" -s "Rsync copy process for day $DAY starting \
|
|
on $THIS_HOST" data_support < $MAILMESSAGEFILE
|
|
|
|
# Start all of the rsync copy sessions at once by looping
|
|
# through each of the mount points and issuing an rsync
|
|
# command in the background, and incrementing a counter.
|
|
|
|
echo "\nStarting a bunch of rsync sessions...\n"
|
|
|
|
# This script copies from the DM filesystems to the
|
|
# Day 1 or Day 2 filesystems on the OLTP servers.
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
for LOC_MP in $(df | grep oradata_dm_[0-2][0-9] | awk '{print $7}')
|
|
do
|
|
# This sed statement changes the "m" to $DAY
|
|
REM_MP=$(echo $LOC_MP | sed s/m/$DAY/g)
|
|
time rsync -avz ${LOC_MP}/ ${M}:${REM_MP} 2>&1 &
|
|
(( TOTAL = TOTAL + 1 ))
|
|
done
|
|
done
|
|
|
|
# Sleep a few seconds before monitoring processes
|
|
|
|
sleep 10
|
|
|
|
# Give some feedback to the end user.
|
|
|
|
REM_SESSIONS=$(ps -ef | grep "rsync -avz" | grep oradata_dm_[0-2][0-9] \
|
|
| grep -v grep | awk '{print $2}' | wc -l)
|
|
|
|
if (( REM_SESSIONS > 0 ))
|
|
then
|
|
echo "\n$REM_SESSIONS of $TOTAL rsync copy sessions require further updating..."
|
|
echo "\nProcessing rsync copies from $THIS_HOST to both $MACHINE_LIST machines"
|
|
echo "\nPlease be patient, this process may take longer than two hours...\n"
|
|
echo "Rsync is running [Start time: $(date)]\c"
|
|
else
|
|
echo "\nAll files appear to be in sync...verifying file sizes...Please wait...\n"
|
|
fi
|
|
|
|
# While the rsync processes are executing this loop
|
|
# will place a . (dot) every 60 seconds as feedback
|
|
# to the end user that the background rsync copy
|
|
# processes are still executing. When the remaining
|
|
# rsync sessions are less than the total number of
|
|
# sessions (normally 36) this loop will give feedback
|
|
# as to the number of remaining rsync session, as well
|
|
# as the elapsed time of the procesing, every 5 minutes.
|
|
|
|
SECONDS=10
|
|
MIN_COUNTER=0
|
|
|
|
until (( REM_SESSIONS == 0 ))
|
|
do
|
|
sleep 60
|
|
echo ".\c"
|
|
REM_SESSIONS=$(ps -ef | grep "rsync -avz" | grep oradata_dm_[0-2][0-9] \
|
|
| grep -v grep | awk '{print $2}' | wc -l)
|
|
if (( REM_SESSIONS < TOTAL ))
|
|
then
|
|
(( MIN_COUNTER = MIN_COUNTER + 1 ))
|
|
if (( MIN_COUNTER >= 5 ))
|
|
then
|
|
MIN_COUNTER=0
|
|
echo "\n$REM_SESSIONS of $TOTAL rsync sessions \
|
|
remaining $(elapsed_time $SECONDS)\c"
|
|
if (( REM_SESSIONS <= $(( TOTAL / 4 )) ))
|
|
then
|
|
echo "\nRemaining rsync sessions include:\n"
|
|
ps -ef | grep "rsync -avz" | grep oradata_dm_[0-2][0-9] \
|
|
| grep -v grep
|
|
echo
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo "\n...Local rsync processing completed on $THIS_HOST...$(date)"
|
|
echo "\n...Checking remote target machines: $MACHINE_LIST..."
|
|
|
|
# Just because the local rsync processes have completed does
|
|
# not mean that the remote rsync processes have completed
|
|
# execution. This loop verifies that all of the remote
|
|
# rsync processes completed execution. The end user will
|
|
# receive feedback if any of the remote processes are running.
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
for LOC_MP in $(df | grep oradata_dm_[0-2][0-9] | awk '{print $7}')
|
|
do
|
|
# This sed statement changes the "m" to $DAY
|
|
REM_MP=$(echo $LOC_MP | sed s/m/$DAY/g)
|
|
|
|
RPID=$(rsh $M ps -ef | grep rsync | grep $REM_MP \
|
|
| grep -v grep | awk '{print $2}')
|
|
|
|
until [ -z "$RPID" ]
|
|
do
|
|
echo "rsync is processing ${REM_MP} on ${M}...sleeping one minute..."
|
|
sleep 60
|
|
RPID=$(rsh $M ps -ef | grep rsync | grep $REM_MP \
|
|
| grep -v grep | awk '{print $2}')
|
|
done
|
|
done
|
|
done
|
|
|
|
echo "\n...Remote rsync processing completed $(date)\n"
|
|
|
|
# Verify the copy process
|
|
|
|
verify_copy
|
|
if (( $? != 0 ))
|
|
then
|
|
exit 3
|
|
fi
|
|
|
|
# Write to the $COMPLETE_FILE to signal the Oracle DBAs
|
|
# that the copy has completed
|
|
|
|
echo "Rsync copy from $THIS_HOST to machines $MACHINE_LIST \
|
|
completed successfully $(date)" | tee -a $COMPLETE_FILE
|
|
|
|
# Remove the ready to run file
|
|
|
|
rm -f $READYTORUN_FILE >/dev/null 2>&1
|
|
|
|
# Notify completion by email
|
|
|
|
echo "Rsync Copy for Day $DAY Completed Successful Execution \
|
|
on $THIS_HOST $(date)\n" > $MAILMESSAGEFILE
|
|
elapsed_time $SECONDS >> $MAILMESSAGEFILE
|
|
mailx -r "$EMAIL_FROM" -s "Rsync copy for day $DAY completed \
|
|
successfully on $THIS_HOST" data_support < $MAILMESSAGEFILE
|
|
|
|
echo "\nRsync copy completed $(date)"
|
|
|
|
echo "\n[[ $THIS_SCRIPT completed execution $(date) ]]\n"
|
|
|
|
exit 0
|
|
|
|
} | 2>&1 tee -a $LOGFILE
|
|
|
|
###############################################
|
|
# END OF SCRIPT
|
|
###############################################
|