rickshaw-post-process-bench

#!/usr/bin/perl
# -*- mode: perl; indent-tabs-mode: nil; perl-indent-level: 4 -*-
# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=perl
#
# Author: Andrew Theurer
#
# Rickshaw-post-process-bench will run the benchmark-specific post-process script
# for every iteration/sample/client|server.  This is required after running
# rickshaw-run, which runs the benchmarks and puts the result in --base-run-dir

use strict;
use warnings;
use Cwd;
use Data::UUID;
use File::pushd;
use File::Basename;
use File::Temp qw(tempdir);
use File::Copy;
use File::Path qw(make_path);
use JSON::XS;
use JSON::Validator;
use Data::Dumper;

BEGIN {
    if (!(exists $ENV{'TOOLBOX_HOME'} && -d "$ENV{'TOOLBOX_HOME'}/perl")) {
        print "This script requires libraries that are provided by the toolbox project.\n";
        print "Toolbox can be acquired from https://github.com/perftool-incubator/toolbox and\n";
        print "then use 'export TOOLBOX_HOME=/path/to/toolbox' so that it can be located.\n";
        exit 1;
    }
}
use lib "$ENV{'TOOLBOX_HOME'}/perl";
use toolbox::json;
use toolbox::logging;

$toolbox::logging::debug = 0;

my $ug = Data::UUID->new;
my %run; # A multi-dimensional, nested hash, schema TBD
         # This hash documents what was run.

my $base_run_dir;
my $run_file;    # 'rickshaw-run.json' containing all configuration data
                 # (generated by 'rickshaw-run' once a run is complete)
my $result_file; # 'rickshaw-result.json' containing all configuration and result data
                 # (generated by this script)
my $file_rc;
my %ids_to_benchmark;
my %benchmark_to_ids;
my %bench_dirs;

sub usage {
    print "\nusage:\n\n";
    print "--base-run-dir  Directory where result data is located for a previous 'rickshaw-run'\n";
}

sub dump_params {
    my $default_role = 'client';

    my $params_ref = shift;
    my $cs_id = shift;
    my $engine = shift // $default_role;

    my $params_str = "";
    my $benchmark = $ids_to_benchmark{$cs_id};

    foreach my $param (@{ $params_ref }) {
        my $arg = $$param{'arg'};
        my $val = $$param{'val'};
        my $bench = $$param{'benchmark'};
        my $id;
        if (exists $$param{'id'}) {
            $id = $$param{'id'};
        }
        # fallback to client role when role is undefined in json
        my $role = $$param{'role'} // $default_role;

        if (! defined $id or (defined $cs_id and $id eq $cs_id)) {
            if ($bench eq $benchmark) {
                # only dump when role=engine or role=all
                if ( $role eq $engine || $role eq 'all') {
                    if (defined $val && length $val) {
                        if (defined $cs_id) {
                            $val =~ s/\%client-id\%/$cs_id/;
                        }
                        $params_str .= " --" . $arg . "=" . $val;
                    } else {
                        $params_str .= " --" . $arg;
                    }
                }
            }
        }
    }
    $params_str =~ s/^\s//;
    return $params_str;
}

my $rickshaw_project_dir;
{
    # Get the absolute path of the rickshaw project directory
    my $dir = pushd(dirname($0));
    $rickshaw_project_dir = getcwd();
}
my $bench_schema_file = $rickshaw_project_dir . "/schema/benchmark.json";
my $bench_metric_schema_file = $rickshaw_project_dir . "/schema/bench-metric.json";
my $run_schema_file = $rickshaw_project_dir . "/schema/run.json";
my $result_schema_file = $rickshaw_project_dir . "/schema/result.json";
my %bench_configs;

# Process the cmdline params
while (scalar @ARGV > 0) {
    my $p = shift @ARGV;
    debug_log(sprintf "processing \@ARGV, param: [%s]\n", $p);
    my $arg;
    my $val;

    if ( $p =~ /^\-\-(\S+)/ ) {
        $arg = $1;
        if ( $arg =~ /^(\S+)=(.*)/ ) { # '--arg=val'
            $arg = $1;
            $val = $2;
        } else { # '--arg val'
            $val = shift @ARGV;
        }
    } else {
        print "[ERROR]malformed cmdline parameter: %s\n";
        usage;
        exit 1;
    }
    debug_log(sprintf "processing \@ARGV, arg is: [%s], val is: [%s]\n", $arg, $val);
    if ($arg =~ /^help$/) {
        usage;
        exit 0;
    } elsif ($arg =~ /^base-run-dir$/) {
        debug_log(sprintf "argument: [%s]\n", $arg);
        $base_run_dir = $val;
    } else {
        printf "[ERROR]argument not valid: [%s]\n", $arg;
        usage;
        exit 1;
    }
}


# Ensure the run-dir has absolute path
{
    my $dir = pushd($base_run_dir);
    debug_log(sprintf "pushd to [%s]\n", $base_run_dir);
    my $cwd = getcwd();
    debug_log(sprintf "cwd [%s]\n", $cwd);
    $base_run_dir = $cwd;
}
my $config_dir = $base_run_dir . "/config";
my $run_dir = $base_run_dir . "/run";
my $iter_subdir = "iterations";

# Load the existing rickshaw-run.json
$run_file = $run_dir . "/rickshaw-run.json";
debug_log(sprintf "Opening %s\n", $run_file);
($file_rc, my $run_ref) = get_json_file($run_file, $run_schema_file);
if ($file_rc > 0 or ! defined $run_ref) {
    print "Could not open the rickshaw-run.json file\n";
    exit 1;
} else {
    %run = %{ $run_ref };
    # TODO checks for minimum fileds for valid run
}

# Load the bench configs
foreach my $this_bench_dir ( split(/,/, $run{'bench-dir'}) ) {
    my $benchmark_name;
    my $bench_config_file = $this_bench_dir . "/rickshaw.json";
    if (-e $bench_config_file) {
        (my $rc, my $bench_config_ref) = get_json_file($bench_config_file, $bench_schema_file);
        if ($rc > 0 or ! defined $bench_config_ref) {
            print "Could not open the bench config file\n";
            exit 1;
        }
        if (exists $$bench_config_ref{'benchmark'}) {
            $benchmark_name = $$bench_config_ref{'benchmark'};
            $bench_dirs{$benchmark_name} = $this_bench_dir;
            $bench_configs{$benchmark_name} = $bench_config_ref;
        } else {
            print "[ERROR]benchmark was not defined in %s\n", $bench_config_file;
            exit 1;
        }
    } else {
        printf "[ERROR]benchmark subproject config file %s was not found\n", $bench_config_file;
        exit 1;
    }
}

# Build tables to map benchmark to IDs and ID to benchmark
foreach my $benchmark_and_id ( split(/,/, $run{'bench-ids'}) ) {
    (my $bench, my $ids) = split(/:/, $benchmark_and_id);
    my @id_ranges = split(/,/, $ids);
    foreach my $id_range (@id_ranges) {
        if ($id_range =~ /^(\d+)\-(\d+)$/) {
            for (my $id = $1; $id <= $2; $id++) {
                $ids_to_benchmark{$id} = $bench;
                push(@{ $benchmark_to_ids{$bench} }, $id);
            }
        } elsif ($id_range =~ /^(\d+)$/) {
            push(@{ $benchmark_to_ids{$bench} }, $id_range);
            $ids_to_benchmark{$id_range} = $bench;
        } else {
            printf "ID range or number not recognized: %s\n", $id_range;
        }
    }
}


printf "Launching a post-process job for each iteration x sample x [client|server] for %s\n", $run{'benchmark'};
for (my $i = 1; $i <= scalar @{ $run{'iterations'} }; $i++) {
    my $iter_dh;
    my $this_iter_dir = $iter_subdir . "/iteration-" . $i;
    if (! -d $run_dir . "/" . $this_iter_dir) {
        printf "The directory [%s] does not exist\n", $run_dir . "/" . $this_iter_dir;
        next;
    }
    if (opendir($iter_dh, $run_dir . "/" . $this_iter_dir)) {
        my @samp_dirs = grep(/^sample-\d+$/, readdir($iter_dh));
        if (scalar @samp_dirs < $run{'num-samples'}) {
            printf "Iteration %d, [%s] is considered failed because there were not enough passing samples (%d)\n",
                   $i, $this_iter_dir, $run{'num-samples'};
            system("/bin/mv " . $run_dir . "/" . $this_iter_dir . " " . $run_dir . "/" . $this_iter_dir . "-fail");
        }
        my $iter_array_idx = $i -1;
        my @samples;
        my $primary_metric;
        my $primary_period;
            for my $samp_dir (@samp_dirs) {
            my @pids;
            my %sample; # sample data from all clients/servers
            my @cons_periods; # consolidated periods across clients/servers get merged here
            my $this_samp_dir = $this_iter_dir . "/" .  $samp_dir;
            printf "Working on " . $this_samp_dir . "\n";
            if (opendir(my $samp_dh, $run_dir . "/" . $this_samp_dir)) {
                my @cs_names = grep(/^(client|server)$/, readdir($samp_dh));
                for my $cs_name (@cs_names) { 
                    my $cs_name_dir = $this_samp_dir . "/" . $cs_name;
                    if (opendir(my $cs_name_dh, $run_dir . "/" . $cs_name_dir)) {
                        my @cs_ids = grep(/^(\d+)$/, readdir($cs_name_dh));
                        for my $cs_id (@cs_ids) {
                            # The benchmark type is now dependent on the csid
                            my $benchmark = $ids_to_benchmark{$cs_id};
                            my $bench_dir = $bench_dirs{$benchmark};
                            my $pp_cmd = $bench_configs{$benchmark}{'controller'}{'post-script'};
                            $pp_cmd =~ s/\%bench-dir\%/$bench_dir\//g;
                            $pp_cmd =~ s/\%run-dir\%/$run_dir\//g;
                            $pp_cmd =~ s/\%config-dir\%/$config_dir\//g;
                            my $iter_params = dump_params($run{'iterations'}[$i - 1]{'params'}, $cs_id, $cs_name);
                            my $cs_id_dir = $cs_name_dir . "/" . $cs_id;
                            if (-d $run_dir . "/" . $cs_id_dir) {
                                if (my $pid = fork) {
                                    push(@pids, $pid);
                                } else {
                                    my $pushd_dir = pushd($run_dir . "/" . $cs_id_dir);
                                    my $full_cmd = "RS_CS_LABEL=" . $cs_name . "-" . $cs_id . " " . $pp_cmd . " " . $iter_params . " >post-process-output.txt 2>&1";
                                    debug_log(sprintf "About to run: [%s]\n", $full_cmd);
                                    my $rc = system($full_cmd);
                                    if ($rc == -1) {
                                        printf "Failed to execute '%s%s'!\n", $pp_cmd, $iter_params;
                                    } elsif ($rc & 127) {
                                        printf "'%s%s' died with signal %d, %s coredump!\n", $pp_cmd, $iter_params, ($rc & 127), ($rc & 128) ? 'with' : 'without';
                                    } else {
                                        $rc = $rc >> 8;
                                        if ($rc != 0) {
                                        printf "'%s%s' exited with non-zero value %d\n", $pp_cmd, $iter_params, $rc;
                                        }
                                    }
                                    exit;
                                }
                            }
                        }
                    }
                }
            }
            printf "Waiting for %d post-processing jobs to complete\n", scalar @pids;
            while (wait() > -1) {}
            print "Post-processing complete\n";
        }
    }
}