Gitlab Community Edition Instance

Skip to content

GitLab

Commit 3120ba8c authored by Bjørn Nachtwey's avatar Bjørn Nachtwey
Browse files

0.6.10.2 create copy of profile file with Date for timeline anaylsis

parent 3dbfd68b
Hide whitespace changes
Inline Side-by-side
perl/dsmci.pl
View file @ 3120ba8c
......@@ -68,6 +68,7 @@
# 2021-07-15 0.6.9 identify and remove duplicate entries with "SUN" and "SUY" keeping "SUN" if both are given
# 2021-11-29 0.6.10 move evaluation for profiling to child tread, collect more detailed data
# 2021-11-29 0.6.10.1 changed "Bytes inspected" and "Bytes transferred" in prof file to "GBytes"
# 2022-01-24 0.6.10.2 create copy of profile file with Date for timeline anaylsis
#
# important notes
#
......@@ -86,6 +87,7 @@ use strict;
use warnings;
use File::Spec::Functions;
use File::Find::Rule;
use File::Copy;
use Time::Piece;
use Fcntl qw(:flock);
......@@ -118,28 +120,28 @@ use constant RETURNPIDFOUND => 15; # stopped due to existing pid file
use constant RETURNNOTHREAD => 21; # cannot start new threads
my $date; # date for statistics file | localtime->strftime ('%F') does not work on windows
my $starttime = time(); # time in seconds since Jan 1, 1970
my $starttimestring; # starting time | localtime->strftime ('%F %R') does not work on windows
my $endtime; #
my $endtimestring; #
my $min; # Minute part of localtime array
my $hour; # hour part of localtime array
my $mday; # day of month part of localtime array
my $mon; # month part of localtime array
my $year; # year part of localtime array
my $starttime = time(); # time in seconds since Jan 1, 1970
my $starttimestring; # starting time | localtime->strftime ('%F %R') does not work on windows
my $endtime; #
my $endtimestring; #
my $min; # Minute part of localtime array
my $hour; # hour part of localtime array
my $mday; # day of month part of localtime array
my $mon; # month part of localtime array
my $year; # year part of localtime array
my $total_elaped_time = 0.0; # total processing time in seconds
my $datatransfertime = 0.0; # total data transfer time in seconds "Data transfer time: 0.00 sec"
my $wallclocktime; # total wallclock time in seconds
my $speedup; # ratio of processing time and wallclock time
my $dttratio = 0.0; # ratio between datatransfer time and total time
my $wallclocktime; # total wallclock time in seconds
my $speedup; # ratio of processing time and wallclock time
my $dttratio = 0.0; # ratio between datatransfer time and total time
my $pidfile; # name of the pid file
my $ppid; # parent process id (this script originally)
my $cpid; # child process id (copy of this script)
my $startpath; # pathroot where the backup should start from
my @startpaths = undef; # array for multiple startpaths
my $actpath; # starting path actually processed
my $actpathdir; # name of the actually procces path for logging
my $actpath; # starting path actually processed
my $actpathdir; # name of the actually procces path for logging
my $maxdepth = 3; # number of directory level to dive into, right here all folders are processed with "-su=y"
my $depth = ($maxdepth - 1); # just one level above $maxdepth, down to here all folders are processed with "-su=n"
......@@ -147,13 +149,14 @@ my $cfgfilename = "dsmci.cfg"; # name of config file, should be located in curr
my $log_filename; # filename for logging
my $err_filename; # filename for error logging
my @errorlines; # An Array holding all the logged warnings, errors and severe errors
my $errorlogfilename; # filename for collected error messages
my $errorlogfilename; # filename for collected error messages
my $globallog_filename; # filename for collected status logs
my @logfiles; # array for names of all logfiles
my $logfile; # name of single logfile
my $childlogapx = ".child.log"; # Appendix of child files
my $statfilename; # Path and Name of statistics file
my $proffilename; # Path and Name of profiling file
my $todproffilename; # Path and Name of profiling file with date
my $profapx = ".prof"; # Appendix of profiling file
my @proftimes; # an Array holding all subfolders and processing times
my @profdirs; # an Array holding all subfolders with profiling times available
......@@ -189,15 +192,15 @@ my $sevecount = 0; # amount of ANS....S messages
my $SooScount = 0; # amount of ANS1329S "Server out of Space" messages
my $returnval = -1;
# due to the ISP statistics
my $total_objects_inspected = 0; # "Total number of objects inspected"
my $total_objects_backed_up = 0; # "Total number of objects backed up"
my $total_objects_updated = 0; # "Total number of objects updated"
my $total_objects_deleted = 0; # "Total number of objects deleted"
my $total_objects_expired = 0; # "Total number of objects expired"
my $total_objects_failed = 0; # "Total number of objects failed"
my $total_bytes_inspected = 0; # "Total number of bytes inspected"
my $total_bytes_transferred = 0; # "Total number of bytes transferred"
my $total_transfer_time = 0; # "Total transfer time"
my $total_objects_inspected = 0; # "Total number of objects inspected"
my $total_objects_backed_up = 0; # "Total number of objects backed up"
my $total_objects_updated = 0; # "Total number of objects updated"
my $total_objects_deleted = 0; # "Total number of objects deleted"
my $total_objects_expired = 0; # "Total number of objects expired"
my $total_objects_failed = 0; # "Total number of objects failed"
my $total_bytes_inspected = 0; # "Total number of bytes inspected"
my $total_bytes_transferred = 0; # "Total number of bytes transferred"
my $total_transfer_time = 0; # "Total transfer time"
#filehandles
my $PPIDFILE;
......@@ -313,6 +316,11 @@ foreach $actpath (@startpaths)
# some preparation
#
###############################################################################
# get from localtime()
(undef, $min, $hour, $mday, $mon, $year, undef, undef, undef) = localtime();
# sec min hour mday mon year wday yday isdst
$date = sprintf "%d-%2.2d-%2.2d", $year+1900, $mon+1, $mday;
$starttimestring = sprintf "%d-%2.2d-%2.2d %2.2d:%2.2d", $year+1900, $mon+1, $mday, $hour, $min;
# set name of PID file
$pidfile = "dsmci.pid"; # set path and name of timestampfile
......@@ -321,6 +329,9 @@ $log_filename = $$ . "log";
$err_filename = $$ . "err";
$globallog_filename = $$ . ".all.log";
$proffilename = File::Spec->canonpath("dsmci".$profapx);
$todproffilename = File::Spec->canonpath($date.".dsmci".$profapx);
if ( $pidcheck == TRUE )
{
## check for running processes like this
......@@ -338,11 +349,7 @@ if ( $pidcheck == TRUE )
}
}
# get from localtime()
(undef, $min, $hour, $mday, $mon, $year, undef, undef, undef) = localtime();
# sec min hour mday mon year wday yday isdst
$date = sprintf "%d-%2.2d-%2.2d", $year+1900, $mon+1, $mday;
$starttimestring = sprintf "%d-%2.2d-%2.2d %2.2d:%2.2d", $year+1900, $mon+1, $mday, $hour, $min;
###############################################################################
###############################################################################
......@@ -363,7 +370,7 @@ foreach $actpath (@startpaths)
$rule->maxdepth($maxdepth);
$rule->mindepth($maxdepth);
$rule->not($rule->new->symlink);
my @tsuyd =$rule->in($actpath);
my @tsuyd = $rule->in($actpath);
# add su=y information to each path entry
foreach $item (@tsuyd)
......@@ -376,7 +383,7 @@ foreach $actpath (@startpaths)
$rule->maxdepth($depth);
$rule->mindepth(0);
$rule->not($rule->new->symlink);
my @tsund = $rule->in($actpath);
my @tsund = $rule->in($actpath);
# add su=y information to each path entry
foreach $item (@tsund)
{
......@@ -395,14 +402,14 @@ shift @allpaths;
###############################################################################
# sort list to put "similar" entries next together
my @sortlist = sort @allpaths;
my @sortlist = sort @allpaths;
# undef @allpaths, list will be recreated when after duplicates are removed
undef @allpaths;
# look for duplicate entries
# => this also reduces all entries to just one entity with SUN and SUY
my %seen;
my %seen;
foreach $item (@sortlist)
{
......@@ -415,18 +422,15 @@ foreach $item (@sortlist)
}
}
###############################################################################
#
# read profiling
# read last profiling
#
###############################################################################
$proffilename = File::Spec->canonpath("dsmci".$profapx);
## code snipped off to "profiling_v2.pl"
#else take the dirs as they are listed
###############################################################################
###############################################################################
# main loop
......@@ -518,12 +522,12 @@ while (my $line = <$GLOBALLOGFILE>)
my $of ; # objects failed
my $bi ; # bytes inspected
my $bt ; # bytes transfered
my $dtt; # transfer time
my $rc = -1; # return code of child process
my $dtt ; # transfer time
my $rc = -1; # return code of child process
(undef, $et, $dir, $rc, $oi, $ob, $ou, $od, $oe, $of, $bi, $bt, $dtt) = split ';', $line;
$total_elaped_time += $et; # "Summary on elapsed time"
$total_elaped_time += $et; # "Summary on elapsed time"
$total_objects_inspected += $oi; # "Total number of objects inspected"
$total_objects_backed_up += $ob; # "Total number of objects backed up"
$total_objects_updated += $ou; # "Total number of objects updated"
......@@ -566,6 +570,8 @@ foreach $line (@sproflines)
{ printf $PROFFILE "%s\n", $line; }
close $PROFFILE;
# create copy of profline file with actual date in name
copy($proffilename, $todproffilename) or warn "Cannot copy profiling file";
###############################################################################
# write error info log
......@@ -593,7 +599,7 @@ else
$dttratio = $datatransfertime / $total_elaped_time * 100.;
}
$total_elaped_time = convert_time($total_elaped_time);
$statfilename = File::Spec->canonpath($date.".dsmci-stats.txt");
$statfilename = File::Spec->canonpath($date.".dsmci-stats.txt");
open $STATFILE, ">", $statfilename
or warn "cannot open $statfilename";
......@@ -704,19 +710,19 @@ sub dbg_print_array
sub convert_time # (c) 2008 by https://neilang.com/articles/converting-seconds-into-a-readable-format-in-perl/
{
my $ltime = shift;
my $days = int($ltime / 86400);
$ltime -= ($days * 86400);
my $hours = int($ltime / 3600);
$ltime -= ($hours * 3600);
my $minutes = int($ltime / 60);
my $seconds = $ltime % 60;
$days = $days < 1 ? '' : $days .'d ';
$hours = $hours < 1 ? '' : $hours .'h ';
$minutes = $minutes < 1 ? '' : $minutes . 'm ';
$ltime = $days . $hours . $minutes . $seconds . 's';
return $ltime;
my $ltime = shift;
my $days = int($ltime / 86400);
$ltime -= ($days * 86400);
my $hours = int($ltime / 3600);
$ltime -= ($hours * 3600);
my $minutes = int($ltime / 60);
my $seconds = $ltime % 60;
$days = $days < 1 ? '' : $days .'d ';
$hours = $hours < 1 ? '' : $hours .'h ';
$minutes = $minutes < 1 ? '' : $minutes . 'm ';
$ltime = $days . $hours . $minutes . $seconds . 's';
return $ltime;
}
sub in_array
......@@ -799,7 +805,7 @@ sub fork_backup_threads
my $of = 0; # objects failed
my $bi = 0; # bytes inspected
my $bt = 0; # bytes transfered
my $dtt = 0; # transfer time
my $dtt = 0; # transfer time
# some temporary variables
my @temparray; # just a temporary arry for splitting values
......@@ -854,64 +860,64 @@ sub fork_backup_threads
# collect data for profiling log
if ( $line =~ /Total number of objects inspected/ )
{
@temparray = split ':', $line;
$oi = (pop @temparray);
$oi =~ s/,//g; # just to eliminate grouping delimiter
@temparray = split ':', $line;
$oi = (pop @temparray);
$oi =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Total number of objects backed up/)
{
@temparray = split ':', $line;
$ob = (pop @temparray);
$ob =~ s/,//g; # just to eliminate grouping delimiter
@temparray = split ':', $line;
$ob = (pop @temparray);
$ob =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Total number of objects updated/)
{
@temparray = split ':', $line;
$ou = (pop @temparray);
$ou =~ s/,//g; # just to eliminate grouping delimiter
@temparray = split ':', $line;
$ou = (pop @temparray);
$ou =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Total number of objects deleted/)
{
@temparray = split ':', $line;
$od = (pop @temparray);
$od =~ s/,//g; # just to eliminate grouping delimiter
@temparray = split ':', $line;
$od = (pop @temparray);
$od =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Total number of objects expired/)
{
@temparray = split ':', $line;
$oe = (pop @temparray);
$oe =~ s/,//g; # just to eliminate grouping delimiter
@temparray = split ':', $line;
$oe = (pop @temparray);
$oe =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Total number of objects failed/)
{
@temparray = split ':', $line;
$of = (pop @temparray);
$of =~ s/,//g; # just to eliminate grouping delimiter
@temparray = split ':', $line;
$of = (pop @temparray);
$of =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Total number of bytes inspected/)
{
@temparray = split ':', $line;
@temparray = split ':', $line;
($bi, $unit) = split ' ', $temparray[-1];
$bi =~ s/,//g; # just to eliminate grouping delimiter
$bi *= get_unit_mupliplier($unit);
$bi =~ s/,//g; # just to eliminate grouping delimiter
$bi *= get_unit_mupliplier($unit);
}
elsif ($line =~ /Total number of bytes transferred/)
{
@temparray = split ':', $line;
@temparray = split ':', $line;
($bt, $unit) = split ' ', $temparray[-1];
$bt =~ s/,//g; # just to eliminate grouping delimiter
$bt *= get_unit_mupliplier($unit);
$bt =~ s/,//g; # just to eliminate grouping delimiter
$bt *= get_unit_mupliplier($unit);
}
elsif ($line =~ /Data transfer time/)
{
@temparray = split ':', $line;
@temparray = split ':', $line;
($dtt, $unit) = split ' ', $temparray[-1];
$dtt =~ s/,//g; # just to eliminate grouping delimiter
$dtt =~ s/,//g; # just to eliminate grouping delimiter
}
elsif ($line =~ /Elapsed processing time/)
{
(undef, $thour, $tmin, $tsec) = split ':', $line;
$et = $tsec + (60 * $tmin) + (3600 * $thour);
$et = $tsec + (60 * $tmin) + (3600 * $thour);
}
}
# print line for profiling log
......@@ -930,7 +936,7 @@ sub fork_backup_threads
# remove original child log file
unlink $log_filename;
# return rc and finish thread
# return rc and finish thread
exit $childreturnvalue;
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment