409 lines
11 KiB
Bash
Executable File
409 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# SCRIPT: generic_FS_rsync_copy.bash
|
|
# AUTHOR: Randy Michael
|
|
# DATE: 1/14/2008
|
|
# REV: 1.2.1
|
|
#
|
|
# PURPOSE: This script is used to replicate
|
|
#
|
|
#
|
|
# EXIT CODES:
|
|
# 0 ==> Normal execution.
|
|
# 2 ==> Remote host is not pingable.
|
|
# 3 ==> Copy verification failed. One or
|
|
# more local files are not the same
|
|
# size as the remote files.
|
|
# 4 ==> No machines are defined to copy
|
|
# data to.
|
|
# 5 ==> Exited on a trapped signal.
|
|
#
|
|
# set -x # Uncomment to debug this script
|
|
#
|
|
# set -n # Uncomment to check the script.s syntax
|
|
# # without any execution. Do not forget to
|
|
# # recomment this line!
|
|
#
|
|
########################################
|
|
# REVISION LIST:
|
|
########################################
|
|
#
|
|
# IF YOU MODIFY THIS SCRIPT, DOCUMENT THE CHANGE(S)!!!
|
|
#
|
|
########################################
|
|
#
|
|
# Revised by:
|
|
# Revision Date:
|
|
# Revision:
|
|
#
|
|
##############################################
|
|
# DEFINE FILES AND GLOBAL VARIABLES HERE
|
|
##############################################
|
|
|
|
# Define the target machines to copy data to.
|
|
# To specify more than one host enclose the
|
|
# hostnames in double quotes and put at least
|
|
# one space between each hostname
|
|
#
|
|
# EXAMPLE: MACHINE_LIST="fred yogi booboo"
|
|
|
|
MACHINE_LIST="fred"
|
|
|
|
# The FS_PATTERN variable defines the regular expression
|
|
# matching the filesystems we want to replicate with rsync.
|
|
|
|
FS_PATTERN="/data0[1-5]"
|
|
|
|
# Add /usr/local/bin to the PATH
|
|
|
|
export PATH=$PATH:/usr/local/bin
|
|
|
|
# Define the directory containing all of the
|
|
# output files this shell script will produce
|
|
|
|
WORK_DIR=/usr/local/bin
|
|
|
|
# Define the rsync script log file
|
|
|
|
LOGFILE=${WORK_DIR}/generic_FS_rsync_copy.log
|
|
|
|
# Capture the shell script file name
|
|
|
|
THIS_SCRIPT=$(basename $0)
|
|
|
|
# Initialize the background process ID list
|
|
# variable to NULL
|
|
|
|
BG_PID_LIST=
|
|
|
|
# Define the file containing a list of files to verify
|
|
# the sizes match locally and remotely
|
|
|
|
LS_FILES=ls_output.dat
|
|
>$LS_FILES
|
|
|
|
# Initialize the TOTAL variable to 0
|
|
|
|
TOTAL=0
|
|
|
|
# Query the system for the hostname
|
|
|
|
THIS_HOST=$(hostname)
|
|
|
|
# Query the system for the UNIX flavor
|
|
|
|
THIS_OS=$(uname)
|
|
|
|
##############################################
|
|
# DEFINE FUNCTIONS HERE
|
|
##############################################
|
|
|
|
verify_copy ()
|
|
{
|
|
# set -x
|
|
|
|
MYLOGFILE=${WORK_DIR}/verify_rsync_copy_day.log
|
|
>$MYLOGFILE
|
|
ERROR=0
|
|
|
|
# Enclose this loop so we can redirect output to the log file
|
|
# with one assignment at the bottom of the function
|
|
|
|
{
|
|
|
|
# Put a header for the verification log file
|
|
|
|
echo -e "\nRsync copy verification between $THIS_HOST and machine(s) \
|
|
$MACHINE_LIST\n\n" >> $MYLOGFILE
|
|
|
|
# Loop through each machine in the $MACHINE_LIST
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
# Loop through each filesystem that matches the regular
|
|
# regular expression point to by "$FS_PATTERN". It is
|
|
# important that this variable is enclosed within double
|
|
# quotes. Also note the column for the mount point is 6;
|
|
# in AIX this should be changed to 7.
|
|
|
|
for MP in $(df | grep "$FS_PATTERN" | awk '{print $6}')
|
|
do
|
|
# For each filesystem mount point, $MP, execute a
|
|
# find command to find all files in the filesystem.
|
|
# and store them in the $LS_FILES file. We will use this
|
|
# file list to verify the file sizes on the remote
|
|
# machines match the file sizes on the local machine.
|
|
|
|
find $MP -type f > $LS_FILES
|
|
|
|
# Loop through the file list
|
|
|
|
for FL in $(cat $LS_FILES)
|
|
do
|
|
# Find the local file size
|
|
LOC_FS=$(ls -l $FL | awk '{print $5}' 2>&1)
|
|
# Find the remote file size using a remote shell command
|
|
REM_FS=$(rsh $M ls -l $FL | awk '{print $5}' 2>&1)
|
|
echo "Checking File: $FL"
|
|
echo -e "Local $THIS_HOST size:\t$LOC_FS"
|
|
echo -e "Remote $M size:\t$REM_FS"
|
|
|
|
# Ensure the file sizes match
|
|
if [ "$LOC_FS" -ne "$REM_FS" ]
|
|
then
|
|
echo "ERROR: File size mismatch between $THIS_HOST and $M"
|
|
echo "File is: $FL"
|
|
ERROR=1
|
|
fi
|
|
done
|
|
done
|
|
done
|
|
|
|
if (( ERROR != 0 ))
|
|
then
|
|
# Record the failure in the log file
|
|
echo -e "\n\nRSYNC ERROR: $THIS_HOST Rsync copy failed...file size mismatch...\n\n" | tee -a $MYLOGFILE
|
|
echo -e "\nERROR: Rsync copy Failed!"
|
|
echo -e "\n\nCheck log file: $MYLOGFILE\n\n"
|
|
echo -e "\n...Exiting...\n"
|
|
return 3
|
|
else
|
|
echo -e "\nSUCCESS: Rsync copy completed successfully..."
|
|
echo -e "\nAll file sizes match...\n"
|
|
fi
|
|
} | 2>&1 tee -a $MYLOGFILE
|
|
|
|
}
|
|
|
|
##############################################
|
|
|
|
elapsed_time ()
|
|
{
|
|
SEC=$1
|
|
(( SEC < 60 )) && echo -e "[Elapsed time: $SEC seconds]\c"
|
|
|
|
(( SEC >= 60 && SEC < 3600 )) && echo -e "[Elapsed time: $(( SEC / 60 )) min $(( SEC % 60 )) sec]\c"
|
|
|
|
(( SEC > 3600 )) && echo -e "[Elapsed time: $(( SEC / 3600 )) hr $(( (SEC % 3600) / 60 )) min $(( (SEC % 3600) % 60 )) sec]\c"
|
|
}
|
|
|
|
##############################################
|
|
# BEGINNING OF MAIN
|
|
##############################################
|
|
|
|
# Enclose the entire main part of the script in
|
|
# curly braces so we can redirect all output of
|
|
# the shell script with a single redirection
|
|
# at the bottom of the script to the $LOGFILE
|
|
|
|
{
|
|
|
|
# Save a copy of the last log file
|
|
|
|
cp -f $LOGFILE ${LOGFILE}.yesterday
|
|
|
|
echo -e "\n[[ $THIS_SCRIPT started execution $(date) ]]\n"
|
|
|
|
# Ensure that target machines are defined
|
|
|
|
if [ -z "$MACHINE_LIST" ]
|
|
then
|
|
echo -e "\nERROR: No machines are defined to copy data to..."
|
|
echo "...Unable to continue...Exiting..."
|
|
exit 4
|
|
fi
|
|
|
|
# Ensure the remote machines are reachable through
|
|
# the network by sending 1 ping.
|
|
|
|
echo -e "Verifying the target machines are pingable...\n"
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
echo "Pinging $M..."
|
|
ping -c1 $M >/dev/null 2>&1
|
|
if (( $? != 0 ))
|
|
then
|
|
echo -e "\nERROR: $M host is not pingable...cannot continue..."
|
|
echo -e "...EXITING...\n"
|
|
exit 2
|
|
else
|
|
echo "Pinging $M succeeded..."
|
|
fi
|
|
done
|
|
|
|
# Start all of the rsync copy sessions at once by looping
|
|
# through each of the mount points and issuing an rsync
|
|
# command in the background, and incrementing a counter.
|
|
|
|
echo -e "\nStarting a bunch of rsync sessions...\n"
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
# Loop through each filesystem that matches the regular
|
|
# expression point to by "$FS_PATTERN". It is
|
|
# important that this variable is enclosed within double
|
|
# quotes. Also note the column for the mount point is 6;
|
|
# in AIX this should be changed to 7.
|
|
|
|
for MP in $(df | grep "$FS_PATTERN" | awk '{print $6}')
|
|
do
|
|
# Start the rsync session in the background
|
|
|
|
time rsync -avz ${MP}/ ${M}:${MP} 2>&1 &
|
|
|
|
# Keep a running total of the number of rsync
|
|
# sessions started
|
|
|
|
(( TOTAL = TOTAL + 1 ))
|
|
done
|
|
done
|
|
|
|
# Sleep a few seconds before monitoring processes
|
|
|
|
sleep 10
|
|
|
|
# Find the number of rsync sessions that are still running
|
|
|
|
REM_SESSIONS=$(ps x | grep "rsync -avz" | grep "$FS_PATTERN" \
|
|
| grep -v grep | awk '{print $1}' | wc -l)
|
|
|
|
# Give some feedback to the end user.
|
|
|
|
if (( REM_SESSIONS > 0 ))
|
|
then
|
|
echo -e "\n$REM_SESSIONS of $TOTAL rsync copy sessions require \
|
|
further updating..."
|
|
echo -e "\nProcessing rsync copies from $THIS_HOST to both \
|
|
$MACHINE_LIST machines"
|
|
echo -e "\nPlease be patient, this process may take a very long \
|
|
time...\n"
|
|
echo -e "Rsync is running [Start time: $(date)]\c"
|
|
else
|
|
echo -e "\nAll files appear to be in sync...verifying file sizes...\
|
|
Please wait...\n"
|
|
fi
|
|
|
|
# While the rsync processes are executing this loop
|
|
# will place a . (dot) every 60 seconds as feedback
|
|
# to the end user that the background rsync copy
|
|
# processes are still executing. When the remaining
|
|
# rsync sessions are less than the total number of
|
|
# sessions (normally 36) this loop will give feedback
|
|
# as to the number of remaining rsync session, as well
|
|
# as the elapsed time of the processing, every 5 minutes..
|
|
|
|
SECONDS=10
|
|
MIN_COUNTER=0
|
|
|
|
# Loop until all of the local rsync sessions have completed
|
|
|
|
until (( REM_SESSIONS == 0 ))
|
|
do
|
|
# sleep 60 seconds between loop iterations
|
|
|
|
sleep 60
|
|
|
|
# Display a dot on the screen every 60 seconds
|
|
|
|
echo -e ".\c"
|
|
|
|
# Find the number of remaining rsync sessions
|
|
|
|
REM_SESSIONS=$(ps x | grep "rsync -avz" | grep "$FS_PATTERN" \
|
|
| grep -v grep | '{print $1}' | wc -l)
|
|
|
|
# Have any of the sessions completed? If so start giving
|
|
# the user updates every 5 minutes, specifying the number
|
|
# remaining rsync sessions and the elapsed time.
|
|
|
|
if (( REM_SESSIONS < TOTAL ))
|
|
then
|
|
# Count every 5 minutes
|
|
(( MIN_COUNTER = MIN_COUNTER + 1 ))
|
|
|
|
# 5 minutes timed out yet?
|
|
if (( MIN_COUNTER >= 5 ))
|
|
then
|
|
# Reset the minute counter
|
|
MIN_COUNTER=0
|
|
|
|
# Update the user with a new progress report
|
|
|
|
echo -e "\n$REM_SESSIONS of $TOTAL rsync sessions \
|
|
remaining $(elapsed_time $SECONDS)\c"
|
|
|
|
# Have three-fourths of the rsync sessions completed?
|
|
|
|
if (( REM_SESSIONS <= $(( TOTAL / 4 )) ))
|
|
then
|
|
# Display the list of remaining rsync sessions
|
|
# that continue to run.
|
|
|
|
echo -e "\nRemaining rsync sessions include:\n"
|
|
ps x | grep "rsync -avz" | grep "$FS_PATTERN" \
|
|
| grep -v grep
|
|
echo
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo -e "\n...Local rsync processing completed on $THIS_HOST...$(date)"
|
|
echo -e "\n...Checking remote target machines: $MACHINE_LIST..."
|
|
|
|
# Just because the local rsync processes have completed does
|
|
# not mean that the remote rsync processes have completed
|
|
# execution. This loop verifies that all of the remote
|
|
# rsync processes completed execution. The end user will
|
|
# receive feedback if any of the remote processes are running.
|
|
|
|
for M in $MACHINE_LIST
|
|
do
|
|
# Loop through each matching filesystem
|
|
|
|
for MP in $(df | grep "$FS_PATTERN" | awk '{print $7}')
|
|
do
|
|
# Find the remote process Ids.
|
|
RPID=$(rsh $M ps x | grep rsync | grep $MP | grep -v grep \
|
|
| awk '{print $1}')
|
|
|
|
# Loop while remote rsync sessions continue to run.
|
|
# NOTE: I have never found remote rsync sessions running
|
|
# after the local sessions have completed. This is just
|
|
# an extra sanity check
|
|
|
|
until [ -z "$RPID" ]
|
|
do
|
|
echo "rsync is processing ${MP} on ${M}...sleeping one minute..."
|
|
sleep 60
|
|
RPID=$(rsh $M ps x | grep rsync | grep $MP | grep -v grep \
|
|
| awk '{print $1}')
|
|
done
|
|
done
|
|
done
|
|
|
|
echo -e "\n...Remote rsync processing completed $(date)\n"
|
|
|
|
# Verify the copy process
|
|
|
|
verify_copy
|
|
|
|
# Check the verify_copy return code
|
|
|
|
if (( $? != 0 ))
|
|
then
|
|
exit 3
|
|
fi
|
|
|
|
echo -e "\nRsync copy completed $(date)"
|
|
|
|
echo -e "\n[[ $THIS_SCRIPT completed execution $(date) ]]\n"
|
|
echo -e "\n[[ Elapsed Time: $(elapsed_time $SECONDS) ]]\n"
|
|
|
|
} | 2>&1 tee -a $LOGFILE
|
|
|
|
###############################################
|
|
# END OF SCRIPT
|
|
###############################################
|