github/verilator
1
0
Fork 1
mirror of https://github.com/verilator/verilator.git synced 2025-01-08 15:47:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tests: Add simulator benchmarking data option for regression tests (#3054)

Browse Source 
This commit adds the '--simbenchmark' option to the regression test compile command.
The option is not intended as a fully-fledged benchmarking infrastructure, but rather a
utility for easily generating cycle- and execution time information when executing a verilated test.

As an example use case, the included test file shows how optimization level is varied across
three different builds+simulations, with the statistics for each run output to the same file in
the output directory.

Future work:
- 'sim_time' in the generated top-level main file should be a parameter.
- Given the above, the test execution script from verilog-sim-benchmark can be integrated
to generate better estimates of cycles/second through varying 'sim_time' over multiple executions.
This commit is contained in:
Morten Borup Petersen 2021-07-01 17:17:55 +02:00 committed by GitHub
parent 708abe0dd1
commit 2c813488f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 104 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
test_regress/driver.pl
View File

@ -697,6 +697,23 @@ sub new {
return $self;
}
sub benchmarksim_filename {
my $self = (ref $_[0] ? shift : $Self);
return $self->{obj_dir}."/$self->{name}_benchmarksim.csv";
}
sub init_benchmarksim {
my $self = (ref $_[0] ? shift : $Self);
# Simulations with benchmarksim enabled append to the same file between runs.
# Test files must ensure a clean benchmark data file before executing tests.
my $filename = $self->benchmarksim_filename();
my $fh = IO::File->new(">".$filename) or die "%Error: $! ".$filename;
print $fh "# Verilator simulation benchmark data\n";
print $fh "# Test name: ".$self->{name}."\n";
print $fh "# Top file: ".$self->{top_filename}."\n";
print $fh "evals, time[s]\n";
}
sub soprint {
my $self = (ref $_[0] ? shift : $Self);
my $str = "$self->{scenario}/$self->{name}: ".join('',@_);
@ -896,6 +913,7 @@ sub compile_vlt_flags {
$self->{savable} = 1 if ($checkflags =~ /-savable\b/);
$self->{coverage} = 1 if ($checkflags =~ /-coverage\b/);
$self->{sanitize} = $opt_sanitize unless exists($self->{sanitize});
$self->{benchmarksim} = 1 if ($param{benchmarksim});
my @verilator_flags = @{$param{verilator_flags}};
unshift @verilator_flags, "--gdb" if $opt_gdb;
@ -1738,6 +1756,10 @@ sub _make_main {
print $fh "#define MAIN_TIME_MULTIPLIER ".($self->{main_time_multiplier} || 1)."\n";
print $fh "#include <memory>\n";
print $fh "#include <fstream>\n" if $self->{benchmarksim};
print $fh "#include <chrono>\n" if $self->{benchmarksim};
print $fh "#include <iomanip>\n" if $self->{benchmarksim};
print $fh "// OS header\n";
print $fh "#include \"verilatedos.h\"\n";
@ -1806,6 +1828,12 @@ sub _make_main {
$set = "topp->";
}
if ($self->{benchmarksim}) {
$fh->print(" std::chrono::time_point<std::chrono::steady_clock> starttime;\n");
$fh->print(" bool warm = false;\n");
$fh->print(" uint64_t n_evals = 0;\n");
}
if ($self->{trace}) {
$fh->print("\n");
$fh->print("#if VM_TRACE\n");
@ -1865,7 +1893,25 @@ sub _make_main {
}
_print_advance_time($self, $fh, 1, $action);
}
if ($self->{benchmarksim}) {
$fh->print(" if (VL_UNLIKELY(!warm)) {\n");
$fh->print(" starttime = std::chrono::steady_clock::now();\n");
$fh->print(" warm = true;\n");
$fh->print(" } else {\n");
$fh->print(" ++n_evals;\n");
$fh->print(" }\n");
}
print $fh " }\n";
if ($self->{benchmarksim}) {
$fh->print(" {\n");
$fh->print(" const std::chrono::duration<double> exec_s = std::chrono::steady_clock::now() - starttime;\n");
$fh->print(" std::ofstream benchfile(\"".$self->benchmarksim_filename()."\", std::ofstream::out | std::ofstream::app);\n");
$fh->print(" benchfile << std::fixed << std::setprecision(9) << n_evals << \",\" << exec_s.count() << std::endl;\n");
$fh->print(" benchfile.close();\n");
$fh->print(" }\n");
}
print $fh " if (!contextp->gotFinish()) {\n";
print $fh ' vl_fatal(__FILE__, __LINE__, "main", "%Error: Timeout; never got a $finish");',"\n";
print $fh " }\n";
@ -2659,6 +2705,12 @@ The equivalent of C<v_flags> and C<v_flags2>, but only for use with
Verilator. If a flag is a standard flag (+incdir for example) v_flags2
should be used instead.
=item benchmarksim
Output the number of model evaluations and execution time of a test to
I<test_output_dir>/I<test_name>_benchmarksim.csv. Multiple invocations
of the same test file will append to to the same .csv file.
=item xsim_flags
=item xsim_flags2

52
test_regress/t/t_benchmarksim.pl Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env perl
if (!$::Driver) { use FindBin; exec("$FindBin::Bin/bootstrap.pl", @ARGV, $0); die; }
# DESCRIPTION: Verilator: Verilog Test driver/expect definition
#
# Copyright 2003 by Wilson Snyder. This program is free software; you
# can redistribute it and/or modify it under the terms of either the GNU
# Lesser General Public License Version 3 or the Perl Artistic License
# Version 2.0.
# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
scenarios(vlt => 1);
# Use any top file
top_filename("t/t_gen_alw.v");
init_benchmarksim();
# As an example, compile and simulate the top file with varying optimization level
my @l_opt = (1,2,3);
foreach my $l_opt (@l_opt) {
compile(
benchmarksim => 1,
v_flags2 => ["-O$l_opt"]
);
execute(
check_finished => 1,
);
}
my $fh = IO::File->new("<".benchmarksim_filename()) or error("Benchmark data file not found");
my $lines = 0;
while (defined(my $line = $fh->getline)) {
if ($line =~ /^#/) { next; }
if ($lines == 0) {
error("Expected header but found $line") if $line ne "evals, time[s]\n";
} else {
my @data = grep {$_ != ""} ($line =~ /(\d*\.?\d*)/g);
error("Expected 2 tokens on line ".$lines." but got ".scalar(@data)) if scalar(@data) != 2;
my $cycles = $data[0];
my $time = $data[1];
error("Invalid data on line ".$lines) if $cycles <= 0.0 || $time <= 0.0;
}
$lines += 1;
}
my $n_lines_expected = scalar(@l_opt) + 1;
error("Expected ".$n_lines_expected." lines but found ".$lines) if int($lines) != int($n_lines_expected);
1;
ok(1);