initial commit

This commit is contained in:
Fabio Scotto di Santolo
2020-07-28 19:28:25 +02:00
commit 4cc88d2f6e
245 changed files with 22820 additions and 0 deletions

824
chapter2/24_ways_to_parse.ksh Executable file
View File

@@ -0,0 +1,824 @@
#!/usr/bin/ksh
#
# SCRIPT: 24_ways_to_parse.ksh
# AUTHOR: Randy Michael
# DATE: 08/15/2007
# REV: 2.5.Q
#
# PURPOSE: This script shows the different ways of reading
# a file line-by-line. Again there is not just one way
# to read a file line-by-line and some are faster than
# others and some are more intuitive than others.
#
# REV LIST:
#
# 03/15/2007 - Randy Michael
# Set each of the while loops up as functions and the timing
# of each function to see which one is the fastest.
#
###############
#
# 08/10/2007 - Randy Michael
# Modified this script to include a total of 24 functions
# to parse a file line-by-line.
#
#######################################################################
#
# NOTE: To output the timing to a file use the following syntax:
#
# 24_ways_to_parse.ksh file_to_process > output_file_name 2>&1
#
# The actual timing data is sent to standard error, file
# descriptor (2), and the function name header is sent
# to standard output, file descriptor (1).
#
#######################################################################
#
# set -n # Uncomment to check command syntax without any execution
# set -x # Uncomment to debug this script
#
INFILE="$1"
OUTFILE=writefile.out
TIMEFILE="/tmp/loopfile.out"
>$TIMEFILE
THIS_SCRIPT=$(basename $0)
######################################
function usage
{
echo "\nUSAGE: $THIS_SCRIPT file_to_process\n"
echo "OR - To send the output to a file use: "
echo "\n$THIS_SCRIPT file_to_process > output_file_name 2>&1 \n"
exit 1
}
######################################
function verify_files
{
diff $INFILE $OUTFILE >/dev/null 2>&1
if (( $? != 0 ))
then
echo "ERROR: $INFILE and $OUTFILE do not match"
ls -l $INFILE $OUTFILE
fi
}
######################################
function cat_while_read_LINE
{
# Method 1
# Zero out the $OUTFILE
>$OUTFILE
cat $INFILE | while read LINE
do
echo "$LINE" >> $OUTFILE
:
done
}
######################################
function while_read_LINE_bottom
{
# Method 2
# Zero out the $OUTFILE
>$OUTFILE
while read LINE
do
echo "$LINE" >> $OUTFILE
:
done < $INFILE
}
######################################
function cat_while_LINE_line
{
# Method 3
# Zero out the $OUTFILE
>$OUTFILE
cat $INFILE | while LINE=`line`
do
echo "$LINE" >> $OUTFILE
:
done
}
######################################
function while_LINE_line_bottom
{
# Method 4
# Zero out the $OUTFILE
>$OUTFILE
while LINE=`line`
do
echo "$LINE" >> $OUTFILE
:
done < $INFILE
}
######################################
function cat_while_LINE_line_cmdsub2
{
# Method 5
# Zero out the $OUTFILE
>$OUTFILE
cat $INFILE | while LINE=$(line)
do
echo "$LINE" >> $OUTFILE
:
done
}
######################################
function while_LINE_line_bottom_cmdsub2
{
# Method 6
# Zero out the $OUTFILE
>$OUTFILE
while LINE=$(line)
do
echo "$LINE" >> $OUTFILE
:
done < $INFILE
}
######################################
for_LINE_cat_FILE ()
{
# Method 7
# Zero out the $OUTFILE
>$OUTFILE
for LINE in `cat $INFILE`
do
echo "$LINE" >> $OUTFILE
:
done
}
######################################
for_LINE_cat_FILE_cmdsub2 ()
{
# Method 8
# Zero out the $OUTFILE
>$OUTFILE
for LINE in $(cat $INFILE)
do
echo "$LINE" >> $OUTFILE
:
done
}
#####################################
while_line_outfile ()
{
# Method 9
# Zero out the $OUTFILE
>$OUTFILE
# This function processes every other
# line of the $INFILE, so do not use
# this method
while read
do
line >>$OUTFILE
:
done < $INFILE
}
#####################################
function while_read_LINE_FD_IN
{
# Method 10
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
while read LINE
do
echo "$LINE" >> $OUTFILE
:
done
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
######################################
function cat_while_read_LINE_FD_OUT
{
# Method 11
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
cat $INFILE | while read LINE
do
echo "$LINE"
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
######################################
function while_read_LINE_bottom_FD_OUT
{
# Method 12
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while read LINE
do
echo "$LINE"
:
done < $INFILE
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
######################################
function while_LINE_line_bottom_FD_OUT
{
# Method 13
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while LINE=`line`
do
echo "$LINE"
:
done < $INFILE
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
######################################
function while_LINE_line_bottom_cmdsub2_FD_OUT
{
# Method 14
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while LINE=$(line)
do
echo "$LINE"
:
done < $INFILE
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
######################################
for_LINE_cat_FILE_FD_OUT ()
{
# Method 15
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
for LINE in `cat $INFILE`
do
echo "$LINE"
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
######################################
for_LINE_cat_FILE_cmdsub2_FD_OUT ()
{
# Method 16
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
for LINE in $(cat $INFILE)
do
echo "$LINE"
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
#####################################
while_line_outfile_FD_IN ()
{
# Method 17
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
# This function processes every other
# line of the $INFILE, so do not use
# this method
while read
do
line >> $OUTFILE
:
done
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
#####################################
while_line_outfile_FD_OUT ()
{
# Method 18
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
# This function processes every other
# line of the $INFILE, so do not use
# this method
while read
do
line
:
done < $INFILE
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
}
#####################################
while_line_outfile_FD_IN_AND_OUT ()
{
# Method 19
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while read
do
line
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
#####################################
function while_LINE_line_FD_IN
{
# Method 20
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
while LINE=`line`
do
echo "$LINE" >> $OUTFILE
:
done
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
######################################
function while_LINE_line_cmdsub2_FD_IN
{
# Method 21
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
while LINE=$(line)
do
echo "$LINE" >> $OUTFILE
:
done
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
######################################
function while_read_LINE_FD_IN_AND_OUT
{
# Method 22
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while read LINE
do
echo "$LINE"
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
######################################
function while_LINE_line_FD_IN_AND_OUT
{
# Method 23
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while LINE=`line`
do
echo "$LINE"
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
######################################
function while_LINE_line_cmdsub2_FD_IN_AND_OUT
{
# Method 24
# Zero out the $OUTFILE
>$OUTFILE
# Associate standard input with file descriptor 3
# and redirect standard input to $INFILE
exec 3<&0
exec 0< $INFILE
# Associate standard output with file descriptor 4
# and redirect standard output to $OUTFILE
exec 4<&1
exec 1> $OUTFILE
while LINE=$(line)
do
echo "$LINE"
:
done
# Restore standard output and close file
# descriptor 4
exec 1<&4
exec 4>&-
# Restore standard input and close file
# descriptor 3
exec 0<&3
exec 3>&-
}
######################################
########### START OF MAIN ############
######################################
# Test the Input
# Looking for exactly one parameter
(( $# == 1 )) || usage
# Does the file exist as a regular file?
[[ -f $1 ]] || usage
echo "\nStarting File Processing of each Method\n"
echo "Method 1:"
echo "function cat_while_read_LINE"
time cat_while_read_LINE
verify_files
sleep 1
echo "\nMethod 2:"
echo "function while_read_LINE_bottom"
time while_read_LINE_bottom
verify_files
sleep 1
echo "\nMethod 3:"
echo "function cat_while_LINE_line"
time cat_while_LINE_line
verify_files
sleep 1
echo "\nMethod 4:"
echo "function while_LINE_line_bottom"
time while_LINE_line_bottom
verify_files
sleep 1
echo "\nMethod 5:"
echo "function cat_while_LINE_line_cmdsub2"
time cat_while_LINE_line_cmdsub2
verify_files
sleep 1
echo "\nMethod 6:"
echo "function while_LINE_line_botton_cmdsub2"
time while_LINE_line_bottom_cmdsub2
verify_files
sleep 1
echo "\nMethod 7:"
echo "function for_LINE_cat_FILE"
time for_LINE_cat_FILE
verify_files
sleep 1
echo "\nMethod 8:"
echo "function for_LINE_cat_FILE_cmdsub2"
time for_LINE_cat_FILE_cmdsub2
verify_files
sleep 1
echo "\nMethod 9:"
echo "function while_line_outfile"
time while_line_outfile
verify_files
sleep 1
echo "\nMethod 10:"
echo "function while_read_LINE_FD_IN"
time while_read_LINE_FD_IN
verify_files
sleep 1
echo "\nMethod 11:"
echo "function cat_while_read_LINE_FD_OUT"
time cat_while_read_LINE_FD_OUT
verify_files
sleep 1
echo "\nMethod 12:"
echo "function while_read_LINE_bottom_FD_OUT"
time while_read_LINE_bottom_FD_OUT
verify_files
sleep 1
echo "\nMethod 13:"
echo "function while_LINE_line_bottom_FD_OUT"
time while_LINE_line_bottom_FD_OUT
verify_files
sleep 1
echo "\nMethod 14:"
echo "function while_LINE_line_bottom_cmdsub2_FD_OUT"
time while_LINE_line_bottom_cmdsub2_FD_OUT
verify_files
sleep 1
echo "\nMethod 15:"
echo "function for_LINE_cat_FILE_FD_OUT"
time for_LINE_cat_FILE_FD_OUT
verify_files
sleep 1
echo "\nMethod 16:"
echo "function for_LINE_cat_FILE_cmdsub2_FD_OUT"
time for_LINE_cat_FILE_cmdsub2_FD_OUT
verify_files
sleep 1
echo "\nMethod 17:"
echo "function while_line_outfile_FD_IN"
time while_line_outfile_FD_IN
verify_files
sleep 1
echo "\nMethod 18:"
echo "function while_line_outfile_FD_OUT"
time while_line_outfile_FD_OUT
verify_files
sleep 1
echo "\nMethod 19:"
echo "function while_line_outfile_FD_IN_AND_OUT"
time while_line_outfile_FD_IN_AND_OUT
verify_files
sleep 1
echo "\nMethod 20:"
echo "function while_LINE_line_FD_IN"
time while_LINE_line_FD_IN
verify_files
sleep 1
echo "\nMethod 21:"
echo "function while_LINE_line_cmdsub2_FD_IN"
time while_LINE_line_cmdsub2_FD_IN
verify_files
sleep 1
echo "\nMethod 22:"
echo "function while_read_LINE_FD_IN_AND_OUT"
time while_read_LINE_FD_IN_AND_OUT
verify_files
sleep 1
echo "\nMethod 23:"
echo "function while_LINE_line_FD_IN_AND_OUT"
time while_LINE_line_FD_IN_AND_OUT
verify_files
sleep 1
echo "\nMethod 24:"
echo "function while_LINE_line_cmdsub2_FD_IN_AND_OUT"
time while_LINE_line_cmdsub2_FD_IN_AND_OUT
verify_files

View File

@@ -0,0 +1,19 @@
build_random_line ()
{
# This function extracts random characters
# from the KEYS array by using the RANDOM
# shell variable
C=1
LINE=
until (( C > 79 ))
do
LINE="${LINE}${KEYS[$(($RANDOM % X + 1))]}"
(( C = C + 1 ))
done
# Return the line of random characters
echo "$LINE"
}

17
chapter2/function_elapsed_time Executable file
View File

@@ -0,0 +1,17 @@
elasped_time ()
{
# This elapsed_time function is for Bash shell
SEC=$1
(( SEC < 60 )) && echo -e "[Elasped time: \
$SEC seconds]\c"
(( SEC >= 60 && SEC < 3600 )) && echo -e \
"[Elasped time: $(( SEC / 60 )) min $(( SEC % 60 )) sec]\c"
(( SEC > 3600 )) && echo -e "[Elasped time: \
$(( SEC / 3600 )) hr $(( (SEC % 3600) / 60 )) min \
$(( (SEC % 3600) % 60 )) sec]\c"
}

View File

@@ -0,0 +1,15 @@
load_default_keyboard ()
{
# Loop through each character in the following list and
# append each character to the $CHAR_FILE file. This
# produces a file with one character on each line.
for CHAR in 1 2 3 4 5 6 7 8 9 0 q w e r t y u i o \
p a s d f g h j k l z x c v b n m \
Q W E R T Y U I O P A S D F G H J K L \
Z X C V B N M 0 1 2 3 4 5 6 7 8 9
do
echo "$CHAR" >> $CHAR_FILE
done
}

204
chapter2/random_file.bash Executable file
View File

@@ -0,0 +1,204 @@
#!/bin/bash
#
# SCRIPT: random_file.bash
# AUTHOR: Randy Michael
# DATE: 8/3/2007
# REV: 1.0
# PLATFORM: Not Platform Dependent
#
# PURPOSE: This script is used to create a
# specific size file of random characters.
# The methods used in this script include
# loading an array with alphanumeric
# characters, then using the /dev/random
# character special file to seed the RANDOM
# shell variable, which in turn is used to
# extract a random set of characters from the
# KEYS array to build the OUTFILE of a
# specified MB size.
#
# set -x # Uncomment to debug this script
#
# set -n # Uncomment to check script syntax
# # without aby execution. Do not forget
# # to put the comment back in or the
# # script will never execute.
#
##########################################
# DEFINE FILES AND VARIABLES HERE
##########################################
typeset -i MB_SIZE=$1
typeset -i RN
typeset -i i=1
typeset -i X=0
WORKDIR=/scripts
OUTFILE=${WORKDIR}/largefile.random.txt
>$OUTFILE
THIS_SCRIPT=$(basename $0)
CHAR_FILE=${WORKDIR}/char_file.txt
##########################################
# DEFINE FUNCTIONS HERE
##########################################
build_random_line ()
{
# This function extracts random characters
# from the KEYS array by using the RANDOM
# shell variable
C=1
LINE=
until (( C > 79 ))
do
LINE="${LINE}${KEYS[$(($RANDOM % X + 1))]}"
(( C = C + 1 ))
done
# Return the line of random characters
echo "$LINE"
}
##########################################
elasped_time ()
{
SEC=$1
(( SEC < 60 )) && echo -e "[Elasped time: \
$SEC seconds]\c"
(( SEC >= 60 && SEC < 3600 )) && echo -e \
"[Elasped time: $(( SEC / 60 )) min $(( SEC % 60 )) sec]\c"
(( SEC > 3600 )) && echo -e "[Elasped time: \
$(( SEC / 3600 )) hr $(( (SEC % 3600) / 60 )) min \
$(( (SEC % 3600) % 60 )) sec]\c"
}
##########################################
load_default_keyboard ()
{
# Loop through each character in the following list and
# append each character to the $CHAR_FILE file. This
# produces a file with one character on each line.
for CHAR in 1 2 3 4 5 6 7 8 9 0 q w e r t y u i o \
p a s d f g h j k l z x c v b n m \
Q W E R T Y U I O P A S D F G H J K L \
Z X C V B N M 0 1 2 3 4 5 6 7 8 9
do
echo "$CHAR" >> $CHAR_FILE
done
}
##########################################
usage ()
{
echo -e "\nUSAGE: $THIS_SCRIPT Mb_size"
echo -e "Where Mb_size is the size of the file to build\n"
}
##########################################
# BEGINNING OF MAIN
##########################################
if (( $# != 1 ))
then
usage
exit 1
fi
# Test for an integer value
case $MB_SIZE in
[0-9]) : # Do nothing
;;
*) usage
;;
esac
# Test for the $CHAR_FILE
if [ ! -s "$CHAR_FILE" ]
then
echo -e "\nNOTE: $CHAR_FILE does not esist"
echo "Loading default keyboard data."
echo -e "Creating $CHAR_FILE...\c"
load_default_keyboard
echo "Done"
fi
# Load Character Array
echo -e "\nLoading array with alphanumeric character elements"
while read ARRAY_ELEMENT
do
(( X = X + 1 ))
KEYS[$X]=$ARRAY_ELEMENT
done < $CHAR_FILE
echo "Total Array Character Elements: $X"
# Use /dev/random to seed the shell variable RANDOM
echo "Querying the kernel random number generator for a random seed"
RN=$(dd if=/dev/random count=1 2>/dev/null \
| od -t u4 | awk '{print $2}'| head -n 1)
# The shell variable RANDOM is limited to 32767
echo "Reducing the random seed value to between 1 and 32767"
RN=$(( RN % 32767 + 1 ))
# Initialize RANDOM with a new seed
echo "Assigning a new seed to the RANDOM shell variable"
RANDOM=$RN
echo "Building a $MB_SIZE MB random character file ==> $OUTFILE"
echo "Please be patient, this may take some time to complete..."
echo -e "Executing: .\c"
# Reset the shell SECONDS variable to zero seconds.
SECONDS=0
TOT_LINES=$(( MB_SIZE * 12800 ))
until (( i > TOT_LINES ))
do
build_random_line >> $OUTFILE
(( $(( i % 100 )) == 0 )) && echo -e ".\c"
(( i = i + 1 ))
done
# Capture the total seconds
TOT_SEC=$SECONDS
echo -e "\n\nSUCCESS: $OUTFILE created at $MB_SIZE MB\n"
elasped_time $TOT_SEC
# Calculate the bytes/second file creation rate
(( MB_SEC = ( MB_SIZE * 1024000 ) / TOT_SEC ))
echo -e "\n\nFile Creation Rate: $MB_SEC bytes/second\n"
echo -e "File size:\n"
ls -l $OUTFILE
echo
##########################################
# END OF random_file.bash SCRIPT
##########################################