Gitlab Community Edition Instance

Commit 3120ba8c authored by Bjørn Nachtwey's avatar Bjørn Nachtwey
Browse files

0.6.10.2 create copy of profile file with Date for timeline anaylsis

parent 3dbfd68b
...@@ -68,6 +68,7 @@ ...@@ -68,6 +68,7 @@
# 2021-07-15 0.6.9 identify and remove duplicate entries with "SUN" and "SUY" keeping "SUN" if both are given # 2021-07-15 0.6.9 identify and remove duplicate entries with "SUN" and "SUY" keeping "SUN" if both are given
# 2021-11-29 0.6.10 move evaluation for profiling to child tread, collect more detailed data # 2021-11-29 0.6.10 move evaluation for profiling to child tread, collect more detailed data
# 2021-11-29 0.6.10.1 changed "Bytes inspected" and "Bytes transferred" in prof file to "GBytes" # 2021-11-29 0.6.10.1 changed "Bytes inspected" and "Bytes transferred" in prof file to "GBytes"
# 2022-01-24 0.6.10.2 create copy of profile file with Date for timeline anaylsis
# #
# important notes # important notes
# #
...@@ -86,6 +87,7 @@ use strict; ...@@ -86,6 +87,7 @@ use strict;
use warnings; use warnings;
use File::Spec::Functions; use File::Spec::Functions;
use File::Find::Rule; use File::Find::Rule;
use File::Copy;
use Time::Piece; use Time::Piece;
use Fcntl qw(:flock); use Fcntl qw(:flock);
...@@ -118,28 +120,28 @@ use constant RETURNPIDFOUND => 15; # stopped due to existing pid file ...@@ -118,28 +120,28 @@ use constant RETURNPIDFOUND => 15; # stopped due to existing pid file
use constant RETURNNOTHREAD => 21; # cannot start new threads use constant RETURNNOTHREAD => 21; # cannot start new threads
my $date; # date for statistics file | localtime->strftime ('%F') does not work on windows my $date; # date for statistics file | localtime->strftime ('%F') does not work on windows
my $starttime = time(); # time in seconds since Jan 1, 1970 my $starttime = time(); # time in seconds since Jan 1, 1970
my $starttimestring; # starting time | localtime->strftime ('%F %R') does not work on windows my $starttimestring; # starting time | localtime->strftime ('%F %R') does not work on windows
my $endtime; # my $endtime; #
my $endtimestring; # my $endtimestring; #
my $min; # Minute part of localtime array my $min; # Minute part of localtime array
my $hour; # hour part of localtime array my $hour; # hour part of localtime array
my $mday; # day of month part of localtime array my $mday; # day of month part of localtime array
my $mon; # month part of localtime array my $mon; # month part of localtime array
my $year; # year part of localtime array my $year; # year part of localtime array
my $total_elaped_time = 0.0; # total processing time in seconds my $total_elaped_time = 0.0; # total processing time in seconds
my $datatransfertime = 0.0; # total data transfer time in seconds "Data transfer time: 0.00 sec" my $datatransfertime = 0.0; # total data transfer time in seconds "Data transfer time: 0.00 sec"
my $wallclocktime; # total wallclock time in seconds my $wallclocktime; # total wallclock time in seconds
my $speedup; # ratio of processing time and wallclock time my $speedup; # ratio of processing time and wallclock time
my $dttratio = 0.0; # ratio between datatransfer time and total time my $dttratio = 0.0; # ratio between datatransfer time and total time
my $pidfile; # name of the pid file my $pidfile; # name of the pid file
my $ppid; # parent process id (this script originally) my $ppid; # parent process id (this script originally)
my $cpid; # child process id (copy of this script) my $cpid; # child process id (copy of this script)
my $startpath; # pathroot where the backup should start from my $startpath; # pathroot where the backup should start from
my @startpaths = undef; # array for multiple startpaths my @startpaths = undef; # array for multiple startpaths
my $actpath; # starting path actually processed my $actpath; # starting path actually processed
my $actpathdir; # name of the actually procces path for logging my $actpathdir; # name of the actually procces path for logging
my $maxdepth = 3; # number of directory level to dive into, right here all folders are processed with "-su=y" my $maxdepth = 3; # number of directory level to dive into, right here all folders are processed with "-su=y"
my $depth = ($maxdepth - 1); # just one level above $maxdepth, down to here all folders are processed with "-su=n" my $depth = ($maxdepth - 1); # just one level above $maxdepth, down to here all folders are processed with "-su=n"
...@@ -147,13 +149,14 @@ my $cfgfilename = "dsmci.cfg"; # name of config file, should be located in curr ...@@ -147,13 +149,14 @@ my $cfgfilename = "dsmci.cfg"; # name of config file, should be located in curr
my $log_filename; # filename for logging my $log_filename; # filename for logging
my $err_filename; # filename for error logging my $err_filename; # filename for error logging
my @errorlines; # An Array holding all the logged warnings, errors and severe errors my @errorlines; # An Array holding all the logged warnings, errors and severe errors
my $errorlogfilename; # filename for collected error messages my $errorlogfilename; # filename for collected error messages
my $globallog_filename; # filename for collected status logs my $globallog_filename; # filename for collected status logs
my @logfiles; # array for names of all logfiles my @logfiles; # array for names of all logfiles
my $logfile; # name of single logfile my $logfile; # name of single logfile
my $childlogapx = ".child.log"; # Appendix of child files my $childlogapx = ".child.log"; # Appendix of child files
my $statfilename; # Path and Name of statistics file my $statfilename; # Path and Name of statistics file
my $proffilename; # Path and Name of profiling file my $proffilename; # Path and Name of profiling file
my $todproffilename; # Path and Name of profiling file with date
my $profapx = ".prof"; # Appendix of profiling file my $profapx = ".prof"; # Appendix of profiling file
my @proftimes; # an Array holding all subfolders and processing times my @proftimes; # an Array holding all subfolders and processing times
my @profdirs; # an Array holding all subfolders with profiling times available my @profdirs; # an Array holding all subfolders with profiling times available
...@@ -189,15 +192,15 @@ my $sevecount = 0; # amount of ANS....S messages ...@@ -189,15 +192,15 @@ my $sevecount = 0; # amount of ANS....S messages
my $SooScount = 0; # amount of ANS1329S "Server out of Space" messages my $SooScount = 0; # amount of ANS1329S "Server out of Space" messages
my $returnval = -1; my $returnval = -1;
# due to the ISP statistics # due to the ISP statistics
my $total_objects_inspected = 0; # "Total number of objects inspected" my $total_objects_inspected = 0; # "Total number of objects inspected"
my $total_objects_backed_up = 0; # "Total number of objects backed up" my $total_objects_backed_up = 0; # "Total number of objects backed up"
my $total_objects_updated = 0; # "Total number of objects updated" my $total_objects_updated = 0; # "Total number of objects updated"
my $total_objects_deleted = 0; # "Total number of objects deleted" my $total_objects_deleted = 0; # "Total number of objects deleted"
my $total_objects_expired = 0; # "Total number of objects expired" my $total_objects_expired = 0; # "Total number of objects expired"
my $total_objects_failed = 0; # "Total number of objects failed" my $total_objects_failed = 0; # "Total number of objects failed"
my $total_bytes_inspected = 0; # "Total number of bytes inspected" my $total_bytes_inspected = 0; # "Total number of bytes inspected"
my $total_bytes_transferred = 0; # "Total number of bytes transferred" my $total_bytes_transferred = 0; # "Total number of bytes transferred"
my $total_transfer_time = 0; # "Total transfer time" my $total_transfer_time = 0; # "Total transfer time"
#filehandles #filehandles
my $PPIDFILE; my $PPIDFILE;
...@@ -313,6 +316,11 @@ foreach $actpath (@startpaths) ...@@ -313,6 +316,11 @@ foreach $actpath (@startpaths)
# some preparation # some preparation
# #
############################################################################### ###############################################################################
# get from localtime()
(undef, $min, $hour, $mday, $mon, $year, undef, undef, undef) = localtime();
# sec min hour mday mon year wday yday isdst
$date = sprintf "%d-%2.2d-%2.2d", $year+1900, $mon+1, $mday;
$starttimestring = sprintf "%d-%2.2d-%2.2d %2.2d:%2.2d", $year+1900, $mon+1, $mday, $hour, $min;
# set name of PID file # set name of PID file
$pidfile = "dsmci.pid"; # set path and name of timestampfile $pidfile = "dsmci.pid"; # set path and name of timestampfile
...@@ -321,6 +329,9 @@ $log_filename = $$ . "log"; ...@@ -321,6 +329,9 @@ $log_filename = $$ . "log";
$err_filename = $$ . "err"; $err_filename = $$ . "err";
$globallog_filename = $$ . ".all.log"; $globallog_filename = $$ . ".all.log";
$proffilename = File::Spec->canonpath("dsmci".$profapx);
$todproffilename = File::Spec->canonpath($date.".dsmci".$profapx);
if ( $pidcheck == TRUE ) if ( $pidcheck == TRUE )
{ {
## check for running processes like this ## check for running processes like this
...@@ -338,11 +349,7 @@ if ( $pidcheck == TRUE ) ...@@ -338,11 +349,7 @@ if ( $pidcheck == TRUE )
} }
} }
# get from localtime()
(undef, $min, $hour, $mday, $mon, $year, undef, undef, undef) = localtime();
# sec min hour mday mon year wday yday isdst
$date = sprintf "%d-%2.2d-%2.2d", $year+1900, $mon+1, $mday;
$starttimestring = sprintf "%d-%2.2d-%2.2d %2.2d:%2.2d", $year+1900, $mon+1, $mday, $hour, $min;
############################################################################### ###############################################################################
############################################################################### ###############################################################################
...@@ -363,7 +370,7 @@ foreach $actpath (@startpaths) ...@@ -363,7 +370,7 @@ foreach $actpath (@startpaths)
$rule->maxdepth($maxdepth); $rule->maxdepth($maxdepth);
$rule->mindepth($maxdepth); $rule->mindepth($maxdepth);
$rule->not($rule->new->symlink); $rule->not($rule->new->symlink);
my @tsuyd =$rule->in($actpath); my @tsuyd = $rule->in($actpath);
# add su=y information to each path entry # add su=y information to each path entry
foreach $item (@tsuyd) foreach $item (@tsuyd)
...@@ -376,7 +383,7 @@ foreach $actpath (@startpaths) ...@@ -376,7 +383,7 @@ foreach $actpath (@startpaths)
$rule->maxdepth($depth); $rule->maxdepth($depth);
$rule->mindepth(0); $rule->mindepth(0);
$rule->not($rule->new->symlink); $rule->not($rule->new->symlink);
my @tsund = $rule->in($actpath); my @tsund = $rule->in($actpath);
# add su=y information to each path entry # add su=y information to each path entry
foreach $item (@tsund) foreach $item (@tsund)
{ {
...@@ -395,14 +402,14 @@ shift @allpaths; ...@@ -395,14 +402,14 @@ shift @allpaths;
############################################################################### ###############################################################################
# sort list to put "similar" entries next together # sort list to put "similar" entries next together
my @sortlist = sort @allpaths; my @sortlist = sort @allpaths;
# undef @allpaths, list will be recreated when after duplicates are removed # undef @allpaths, list will be recreated when after duplicates are removed
undef @allpaths; undef @allpaths;
# look for duplicate entries # look for duplicate entries
# => this also reduces all entries to just one entity with SUN and SUY # => this also reduces all entries to just one entity with SUN and SUY
my %seen; my %seen;
foreach $item (@sortlist) foreach $item (@sortlist)
{ {
...@@ -415,18 +422,15 @@ foreach $item (@sortlist) ...@@ -415,18 +422,15 @@ foreach $item (@sortlist)
} }
} }
############################################################################### ###############################################################################
# #
# read profiling # read last profiling
# #
############################################################################### ###############################################################################
$proffilename = File::Spec->canonpath("dsmci".$profapx);
## code snipped off to "profiling_v2.pl" ## code snipped off to "profiling_v2.pl"
#else take the dirs as they are listed #else take the dirs as they are listed
############################################################################### ###############################################################################
############################################################################### ###############################################################################
# main loop # main loop
...@@ -518,12 +522,12 @@ while (my $line = <$GLOBALLOGFILE>) ...@@ -518,12 +522,12 @@ while (my $line = <$GLOBALLOGFILE>)
my $of ; # objects failed my $of ; # objects failed
my $bi ; # bytes inspected my $bi ; # bytes inspected
my $bt ; # bytes transfered my $bt ; # bytes transfered
my $dtt; # transfer time my $dtt ; # transfer time
my $rc = -1; # return code of child process my $rc = -1; # return code of child process
(undef, $et, $dir, $rc, $oi, $ob, $ou, $od, $oe, $of, $bi, $bt, $dtt) = split ';', $line; (undef, $et, $dir, $rc, $oi, $ob, $ou, $od, $oe, $of, $bi, $bt, $dtt) = split ';', $line;
$total_elaped_time += $et; # "Summary on elapsed time" $total_elaped_time += $et; # "Summary on elapsed time"
$total_objects_inspected += $oi; # "Total number of objects inspected" $total_objects_inspected += $oi; # "Total number of objects inspected"
$total_objects_backed_up += $ob; # "Total number of objects backed up" $total_objects_backed_up += $ob; # "Total number of objects backed up"
$total_objects_updated += $ou; # "Total number of objects updated" $total_objects_updated += $ou; # "Total number of objects updated"
...@@ -566,6 +570,8 @@ foreach $line (@sproflines) ...@@ -566,6 +570,8 @@ foreach $line (@sproflines)
{ printf $PROFFILE "%s\n", $line; } { printf $PROFFILE "%s\n", $line; }
close $PROFFILE; close $PROFFILE;
# create copy of profline file with actual date in name
copy($proffilename, $todproffilename) or warn "Cannot copy profiling file";
############################################################################### ###############################################################################
# write error info log # write error info log
...@@ -593,7 +599,7 @@ else ...@@ -593,7 +599,7 @@ else
$dttratio = $datatransfertime / $total_elaped_time * 100.; $dttratio = $datatransfertime / $total_elaped_time * 100.;
} }
$total_elaped_time = convert_time($total_elaped_time); $total_elaped_time = convert_time($total_elaped_time);
$statfilename = File::Spec->canonpath($date.".dsmci-stats.txt"); $statfilename = File::Spec->canonpath($date.".dsmci-stats.txt");
open $STATFILE, ">", $statfilename open $STATFILE, ">", $statfilename
or warn "cannot open $statfilename"; or warn "cannot open $statfilename";
...@@ -704,19 +710,19 @@ sub dbg_print_array ...@@ -704,19 +710,19 @@ sub dbg_print_array
sub convert_time # (c) 2008 by https://neilang.com/articles/converting-seconds-into-a-readable-format-in-perl/ sub convert_time # (c) 2008 by https://neilang.com/articles/converting-seconds-into-a-readable-format-in-perl/
{ {
my $ltime = shift; my $ltime = shift;
my $days = int($ltime / 86400); my $days = int($ltime / 86400);
$ltime -= ($days * 86400); $ltime -= ($days * 86400);
my $hours = int($ltime / 3600); my $hours = int($ltime / 3600);
$ltime -= ($hours * 3600); $ltime -= ($hours * 3600);
my $minutes = int($ltime / 60); my $minutes = int($ltime / 60);
my $seconds = $ltime % 60; my $seconds = $ltime % 60;
$days = $days < 1 ? '' : $days .'d '; $days = $days < 1 ? '' : $days .'d ';
$hours = $hours < 1 ? '' : $hours .'h '; $hours = $hours < 1 ? '' : $hours .'h ';
$minutes = $minutes < 1 ? '' : $minutes . 'm '; $minutes = $minutes < 1 ? '' : $minutes . 'm ';
$ltime = $days . $hours . $minutes . $seconds . 's'; $ltime = $days . $hours . $minutes . $seconds . 's';
return $ltime; return $ltime;
} }
sub in_array sub in_array
...@@ -799,7 +805,7 @@ sub fork_backup_threads ...@@ -799,7 +805,7 @@ sub fork_backup_threads
my $of = 0; # objects failed my $of = 0; # objects failed
my $bi = 0; # bytes inspected my $bi = 0; # bytes inspected
my $bt = 0; # bytes transfered my $bt = 0; # bytes transfered
my $dtt = 0; # transfer time my $dtt = 0; # transfer time
# some temporary variables # some temporary variables
my @temparray; # just a temporary arry for splitting values my @temparray; # just a temporary arry for splitting values
...@@ -854,64 +860,64 @@ sub fork_backup_threads ...@@ -854,64 +860,64 @@ sub fork_backup_threads
# collect data for profiling log # collect data for profiling log
if ( $line =~ /Total number of objects inspected/ ) if ( $line =~ /Total number of objects inspected/ )
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
$oi = (pop @temparray); $oi = (pop @temparray);
$oi =~ s/,//g; # just to eliminate grouping delimiter $oi =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Total number of objects backed up/) elsif ($line =~ /Total number of objects backed up/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
$ob = (pop @temparray); $ob = (pop @temparray);
$ob =~ s/,//g; # just to eliminate grouping delimiter $ob =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Total number of objects updated/) elsif ($line =~ /Total number of objects updated/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
$ou = (pop @temparray); $ou = (pop @temparray);
$ou =~ s/,//g; # just to eliminate grouping delimiter $ou =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Total number of objects deleted/) elsif ($line =~ /Total number of objects deleted/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
$od = (pop @temparray); $od = (pop @temparray);
$od =~ s/,//g; # just to eliminate grouping delimiter $od =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Total number of objects expired/) elsif ($line =~ /Total number of objects expired/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
$oe = (pop @temparray); $oe = (pop @temparray);
$oe =~ s/,//g; # just to eliminate grouping delimiter $oe =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Total number of objects failed/) elsif ($line =~ /Total number of objects failed/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
$of = (pop @temparray); $of = (pop @temparray);
$of =~ s/,//g; # just to eliminate grouping delimiter $of =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Total number of bytes inspected/) elsif ($line =~ /Total number of bytes inspected/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
($bi, $unit) = split ' ', $temparray[-1]; ($bi, $unit) = split ' ', $temparray[-1];
$bi =~ s/,//g; # just to eliminate grouping delimiter $bi =~ s/,//g; # just to eliminate grouping delimiter
$bi *= get_unit_mupliplier($unit); $bi *= get_unit_mupliplier($unit);
} }
elsif ($line =~ /Total number of bytes transferred/) elsif ($line =~ /Total number of bytes transferred/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
($bt, $unit) = split ' ', $temparray[-1]; ($bt, $unit) = split ' ', $temparray[-1];
$bt =~ s/,//g; # just to eliminate grouping delimiter $bt =~ s/,//g; # just to eliminate grouping delimiter
$bt *= get_unit_mupliplier($unit); $bt *= get_unit_mupliplier($unit);
} }
elsif ($line =~ /Data transfer time/) elsif ($line =~ /Data transfer time/)
{ {
@temparray = split ':', $line; @temparray = split ':', $line;
($dtt, $unit) = split ' ', $temparray[-1]; ($dtt, $unit) = split ' ', $temparray[-1];
$dtt =~ s/,//g; # just to eliminate grouping delimiter $dtt =~ s/,//g; # just to eliminate grouping delimiter
} }
elsif ($line =~ /Elapsed processing time/) elsif ($line =~ /Elapsed processing time/)
{ {
(undef, $thour, $tmin, $tsec) = split ':', $line; (undef, $thour, $tmin, $tsec) = split ':', $line;
$et = $tsec + (60 * $tmin) + (3600 * $thour); $et = $tsec + (60 * $tmin) + (3600 * $thour);
} }
} }
# print line for profiling log # print line for profiling log
...@@ -930,7 +936,7 @@ sub fork_backup_threads ...@@ -930,7 +936,7 @@ sub fork_backup_threads
# remove original child log file # remove original child log file
unlink $log_filename; unlink $log_filename;
# return rc and finish thread # return rc and finish thread
exit $childreturnvalue; exit $childreturnvalue;
} }
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment