Gitlab Community Edition Instance

Skip to content

GitLab

Commit c4db1e39 authored by Bjoern Nachtwey's avatar Bjoern Nachtwey
Browse files

0.2 add analysis of output file for statistics

parent 0f498b68
Hide whitespace changes
Inline Side-by-side
Backup/backup-SNFS.sh
View file @ c4db1e39
#! /bin/bash
##############################################################################
#
# backup-SNFS.sh
......@@ -27,6 +28,7 @@
# date version remark
# 2022-03-xx 0.0 initial coding just using file list
# 2022-03-16 0.1 add some more approaches
# 2022-03-31 0.2 add analysis of output file for statistics
# get current date
......@@ -36,23 +38,39 @@ fsname=$1; # Name of Filesystem as mounted below "/stornext"
bpath=/stornext/$fsname # path to it
kind=$2; # switch for using "-incrbydate" option
optfile=/StorNext-Backup/opt/$fsname.opt
templogfile=/StorNext-Backup/$fsname/backup.log
logfile=/StorNext-Backup/$fsname/$today.backup.log
daysback=10
export LANG=en_US
cd /StorNext-Backup/$fsname/
## function for getting the Multiplier of the data amounts
get_unit_mupliplier ()
{
if [[ $1 =~ "B" ]]; then echo 0.000000001;
elif [[ $1 =~ "KB" ]]; then echo 0.000001;
elif [[ $1 =~ "MB" ]]; then echo 0.001;
elif [[ $1 =~ "GB" ]]; then echo 1.0;
elif [[ $1 =~ "TB" ]]; then echo 1000;
elif [[ $1 =~ "PB" ]]; then echo 1000000;
else echo "(-1)";
fi;
}
# presched operations
rm -f /StorNext-Backup/log/$fsname.sched.log
export LANG=en_US
cd /StorNext-Backup/$fsname/
## do backup
case "$kind" in
I) # Standard incremental backup
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile > $templogfile
# get RC
rc=$?;
;;
B) # incremental backup with "-incrbydate"
/opt/tivoli/tsm/client/ba/bin/dsmc i -incrbydate -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc i -incrbydate -optfile=$optfile > $templogfile
# get RC
rc=$?;
;;
......@@ -68,10 +86,14 @@ F) # use filelist backup
ronfile=$today.ron
delfile=$today.del
incfile=$today.inc
tinfile=$today.tin
expfile=$today.exp
# cat last three day's changelist to input file
cat $(find $bpath -maxdepth 1 -name "*_chglst.json" -mtime -$daysback) > $infile
cat $(find -L $bpath -maxdepth 1 -name "*_chglst.json" -mtime -$daysback) > $infile
# grep "DOmain" options from optfile
domains=$(grep -i DOMAIN $optfile | awk '{print $2}')
# new files
grep -A2 '"CREATE"' $infile | grep '"pathname"' | sed -e 's/,$//' | awk -F '\"' '{print $4}' | grep -v "^/[0-9]*$" | sed -e "s#^#$bpath#" | sort | uniq > $newfile
......@@ -86,14 +108,27 @@ F) # use filelist backup
grep -A2 '"REMOVE"' $infile | grep '"pathname"' | sed -e 's/,$//' | awk -F '\"' '{print $4}' | sed -e "s#^#$bpath#" | sort | uniq > $delfile
# files be processed with "i", but filter for files on the most top level
cat missed_files $newfile $chgfile $rnnfile | sort | uniq > $incfile
# 1st create file with all names
cat missed_files $newfile $chgfile $rnnfile | sort | uniq > $tinfile
# 2nd grep for those fitting on DOmain setting from optfile
while read -r item
do
for ltem in $domains
do
if [[ "$item" =~ "$ltem" ]]
then
echo "$item" >> $incfile
continue 2
fi
done
done < $tinfile
# files to be procced with "expire"
cat $delfile $ronfile | sort | uniq > $expfile
# run Backups
starttime=$(date +%s.%N)
/opt/tivoli/tsm/client/ba/bin/dsmc INCremental -filelist=$incfile -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc INCremental -filelist=$incfile -optfile=$optfile > $templogfile
# get backup rc
rc=$?;
grep "ANS4037E" /StorNext-Backup/log/$fsname.error.log | cut -f2 -d"'" > missed_files
......@@ -103,7 +138,7 @@ F) # use filelist backup
# skip expire rc
# get runtime of all
duration=$(echo "$(date +%s.%N) - $starttime" | bc)
duration=$(echo "$(date +%s.%N) - $starttime" | bc -l)
# get & write stats
echo $today "; " $rc $(wc -l $today.??? | awk '{print "; " $1 }' | tr "\n" " ") ";" $duration >> cfl.stats
......@@ -112,7 +147,7 @@ F) # use filelist backup
rm -f $today.*
;;
*) # fallback to standard incremental if nothing is set
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile
/opt/tivoli/tsm/client/ba/bin/dsmc i -optfile=$optfile > $templogfile
# get RC
rc=$?;
;;
......@@ -137,3 +172,31 @@ then
touch /StorNext-Backup/log/$fsname.error.log
bzip2 /StorNext-Backup/log/$today-$fsname.error.log
fi
if [ -s $templogfile ]
then
OI=$(grep "Total number of objects inspected:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OB=$(grep "Total number of objects backed up:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OU=$(grep "Total number of objects updated:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OR=$(grep "Total number of objects rebound:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OD=$(grep "Total number of objects deleted:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OE=$(grep "Total number of objects expired:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
OF=$(grep "Total number of objects failed:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
BIr=$(grep "Total number of bytes inspected:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
BTr=$(grep "Total number of bytes transferred:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
TT=$(grep "Data transfer time:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//' | sed -e 's/,//g');
WTh=$(grep "Elapsed processing time:" $templogfile | cut -d":" -f2 | sed 's/^[ \t]*//;s/[ \t]*$//');
WTm=$(grep "Elapsed processing time:" $templogfile | cut -d":" -f3 | sed 's/^[ \t]*//;s/[ \t]*$//');
WTs=$(grep "Elapsed processing time:" $templogfile | cut -d":" -f4 | sed 's/^[ \t]*//;s/[ \t]*$//');
# calculate values
unit=$(echo "$(echo $BIr | awk '{print $2}')")
BI=$(echo "$(echo $BIr | awk '{print $1}') * $(get_unit_mupliplier $unit)" | bc -l)
unit=$(echo "$(echo $BTr | awk '{print $2}')")
BT=$(echo "$(echo $BTr | awk '{print $1}') * $(get_unit_mupliplier $unit)" | bc -l)
duration=$(echo "$WTs + 60 * $WTm + 3600 * $WTh" | bc -l);
# write stats
echo $today "; " $rc ";" $OI ";" $OB ";" $OU ";" $OD";" $OE ";" $OF ";" $BI ";" $BT ";" $TT ";" $duration >> backup.stats
#
# rename temporary backup log file
mv $templogfile $logfile
bzip2 $logfile
fi
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment