Gitlab Community Edition Instance

Commit c4db1e39 authored by Bjoern Nachtwey's avatar Bjoern Nachtwey
Browse files

0.2 add analysis of output file for statistics

parent 0f498b68
#! /bin/bash
##############################################################################
#
# backup-SNFS.sh
......@@ -27,6 +28,7 @@
# date version remark
# 2022-03-xx 0.0 initial coding just using file list
# 2022-03-16 0.1 add some more approaches
# 2022-03-31 0.2 add analysis of output file for statistics
# get current date
......@@ -36,23 +38,39 @@ fsname=$1; # Name of Filesystem as mounted below "/stornext"
bpath=/stornext/$fsname # path to it
kind=$2; # switch for using "-incrbydate" option
optfile=/StorNext-Backup/opt/$fsname.opt
templogfile=/StorNext-Backup/$fsname/backup.log
logfile=/StorNext-Backup/$fsname/$today.backup.log
daysback=10
export LANG=en_US
cd /StorNext-Backup/$fsname/
## function for getting the Multiplier of the data amounts
get_unit_mupliplier ()
{
if [[ $1 =~ "B" ]]; then echo 0.000000001;
elif [[ $1 =~ "KB" ]]; then echo 0.000001;
elif [[ $1 =~ "MB" ]]; then echo 0.001;
elif [[ $1 =~ "GB" ]]; then echo 1.0;
elif [[ $1 =~ "TB" ]]; then echo 1000;
elif [[ $1 =~ "PB" ]]; then echo 1000000;
else echo "(-1)";
fi;
}
# presched operations
rm -f /StorNext-Backup/log/$fsname.sched.log
export LANG=en_US
cd /StorNext-Backup/$fsname/
## do backup
case "$kind" in
I) # Standard incremental backup
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile > $templogfile
# get RC
rc=$?;
;;
B) # incremental backup with "-incrbydate"
/opt/tivoli/tsm/client/ba/bin/dsmc i -incrbydate -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc i -incrbydate -optfile=$optfile > $templogfile
# get RC
rc=$?;
;;
......@@ -68,10 +86,14 @@ F) # use filelist backup
ronfile=$today.ron
delfile=$today.del
incfile=$today.inc
tinfile=$today.tin
expfile=$today.exp
# cat last three day's changelist to input file
cat $(find $bpath -maxdepth 1 -name "*_chglst.json" -mtime -$daysback) > $infile
cat $(find -L $bpath -maxdepth 1 -name "*_chglst.json" -mtime -$daysback) > $infile
# grep "DOmain" options from optfile
domains=$(grep -i DOMAIN $optfile | awk '{print $2}')
# new files
grep -A2 '"CREATE"' $infile | grep '"pathname"' | sed -e 's/,$//' | awk -F '\"' '{print $4}' | grep -v "^/[0-9]*$" | sed -e "s#^#$bpath#" | sort | uniq > $newfile
......@@ -86,14 +108,27 @@ F) # use filelist backup
grep -A2 '"REMOVE"' $infile | grep '"pathname"' | sed -e 's/,$//' | awk -F '\"' '{print $4}' | sed -e "s#^#$bpath#" | sort | uniq > $delfile
# files be processed with "i", but filter for files on the most top level
cat missed_files $newfile $chgfile $rnnfile | sort | uniq > $incfile
# 1st create file with all names
cat missed_files $newfile $chgfile $rnnfile | sort | uniq > $tinfile
# 2nd grep for those fitting on DOmain setting from optfile
while read -r item
do
for ltem in $domains
do
if [[ "$item" =~ "$ltem" ]]
then
echo "$item" >> $incfile
continue 2
fi
done
done < $tinfile
# files to be procced with "expire"
cat $delfile $ronfile | sort | uniq > $expfile
# run Backups
starttime=$(date +%s.%N)
/opt/tivoli/tsm/client/ba/bin/dsmc INCremental -filelist=$incfile -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc INCremental -filelist=$incfile -optfile=$optfile > $templogfile
# get backup rc
rc=$?;
grep "ANS4037E" /StorNext-Backup/log/$fsname.error.log | cut -f2 -d"'" > missed_files
......@@ -103,7 +138,7 @@ F) # use filelist backup
# skip expire rc
# get runtime of all
duration=$(echo "$(date +%s.%N) - $starttime" | bc)
duration=$(echo "$(date +%s.%N) - $starttime" | bc -l)
# get & write stats
echo $today "; " $rc $(wc -l $today.??? | awk '{print "; " $1 }' | tr "\n" " ") ";" $duration >> cfl.stats
......@@ -112,7 +147,7 @@ F) # use filelist backup
rm -f $today.*
;;
*) # fallback to standard incremental if nothing is set
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile > $templogfile
# get RC
rc=$?;
;;
......@@ -137,3 +172,31 @@ then
touch /StorNext-Backup/log/$fsname.error.log
bzip2 /StorNext-Backup/log/$today-$fsname.error.log
fi
if [ -s $templogfile ]
then
OI=$(grep "Total number of objects inspected:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OB=$(grep "Total number of objects backed up:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OU=$(grep "Total number of objects updated:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OR=$(grep "Total number of objects rebound:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OD=$(grep "Total number of objects deleted:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OE=$(grep "Total number of objects expired:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OF=$(grep "Total number of objects failed:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
BIr=$(grep "Total number of bytes inspected:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
BTr=$(grep "Total number of bytes transferred:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
TT=$(grep "Data transfer time:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
WTh=$(grep "Elapsed processing time:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//');
WTm=$(grep "Elapsed processing time:" $templogfile | cut -d":" -f3 | sed 's/^[ \t]*//;s/[ \t]*$//');
WTs=$(grep "Elapsed processing time:" $templogfile | cut -d":" -f4 | sed 's/^[ \t]*//;s/[ \t]*$//');
# calculate values
unit=$(echo "$(echo $BIr | awk '{print $2}')")
BI=$(echo "$(echo $BIr | awk '{print $1}') * $(get_unit_mupliplier $unit)" | bc -l)
unit=$(echo "$(echo $BTr | awk '{print $2}')")
BT=$(echo "$(echo $BTr | awk '{print $1}') * $(get_unit_mupliplier $unit)" | bc -l)
duration=$(echo "$WTs + 60 * $WTm + 3600 * $WTh" | bc -l);
# write stats
echo $today "; " $rc ";" $OI ";" $OB ";" $OU ";" $OD";" $OE ";" $OF ";" $BI ";" $BT ";" $TT ";" $duration >> backup.stats
#
# rename temporary backup log file
mv $templogfile $logfile
bzip2 $logfile
fi
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment