diff --git a/test_regress/driver.pl b/test_regress/driver.pl index 679906687..5164fcc2f 100755 --- a/test_regress/driver.pl +++ b/test_regress/driver.pl @@ -697,6 +697,23 @@ sub new { return $self; } +sub benchmarksim_filename { + my $self = (ref $_[0] ? shift : $Self); + return $self->{obj_dir}."/$self->{name}_benchmarksim.csv"; +} + +sub init_benchmarksim { + my $self = (ref $_[0] ? shift : $Self); + # Simulations with benchmarksim enabled append to the same file between runs. + # Test files must ensure a clean benchmark data file before executing tests. + my $filename = $self->benchmarksim_filename(); + my $fh = IO::File->new(">".$filename) or die "%Error: $! ".$filename; + print $fh "# Verilator simulation benchmark data\n"; + print $fh "# Test name: ".$self->{name}."\n"; + print $fh "# Top file: ".$self->{top_filename}."\n"; + print $fh "evals, time[s]\n"; +} + sub soprint { my $self = (ref $_[0] ? shift : $Self); my $str = "$self->{scenario}/$self->{name}: ".join('',@_); @@ -896,6 +913,7 @@ sub compile_vlt_flags { $self->{savable} = 1 if ($checkflags =~ /-savable\b/); $self->{coverage} = 1 if ($checkflags =~ /-coverage\b/); $self->{sanitize} = $opt_sanitize unless exists($self->{sanitize}); + $self->{benchmarksim} = 1 if ($param{benchmarksim}); my @verilator_flags = @{$param{verilator_flags}}; unshift @verilator_flags, "--gdb" if $opt_gdb; @@ -1738,6 +1756,10 @@ sub _make_main { print $fh "#define MAIN_TIME_MULTIPLIER ".($self->{main_time_multiplier} || 1)."\n"; print $fh "#include \n"; + print $fh "#include \n" if $self->{benchmarksim}; + print $fh "#include \n" if $self->{benchmarksim}; + print $fh "#include \n" if $self->{benchmarksim}; + print $fh "// OS header\n"; print $fh "#include \"verilatedos.h\"\n"; @@ -1806,6 +1828,12 @@ sub _make_main { $set = "topp->"; } + if ($self->{benchmarksim}) { + $fh->print(" std::chrono::time_point starttime;\n"); + $fh->print(" bool warm = false;\n"); + $fh->print(" uint64_t n_evals = 0;\n"); + } + if ($self->{trace}) { $fh->print("\n"); $fh->print("#if VM_TRACE\n"); @@ -1865,7 +1893,25 @@ sub _make_main { } _print_advance_time($self, $fh, 1, $action); } + if ($self->{benchmarksim}) { + $fh->print(" if (VL_UNLIKELY(!warm)) {\n"); + $fh->print(" starttime = std::chrono::steady_clock::now();\n"); + $fh->print(" warm = true;\n"); + $fh->print(" } else {\n"); + $fh->print(" ++n_evals;\n"); + $fh->print(" }\n"); + } print $fh " }\n"; + + if ($self->{benchmarksim}) { + $fh->print(" {\n"); + $fh->print(" const std::chrono::duration exec_s = std::chrono::steady_clock::now() - starttime;\n"); + $fh->print(" std::ofstream benchfile(\"".$self->benchmarksim_filename()."\", std::ofstream::out | std::ofstream::app);\n"); + $fh->print(" benchfile << std::fixed << std::setprecision(9) << n_evals << \",\" << exec_s.count() << std::endl;\n"); + $fh->print(" benchfile.close();\n"); + $fh->print(" }\n"); + } + print $fh " if (!contextp->gotFinish()) {\n"; print $fh ' vl_fatal(__FILE__, __LINE__, "main", "%Error: Timeout; never got a $finish");',"\n"; print $fh " }\n"; @@ -2659,6 +2705,12 @@ The equivalent of C and C, but only for use with Verilator. If a flag is a standard flag (+incdir for example) v_flags2 should be used instead. +=item benchmarksim + +Output the number of model evaluations and execution time of a test to +I/I_benchmarksim.csv. Multiple invocations +of the same test file will append to to the same .csv file. + =item xsim_flags =item xsim_flags2 diff --git a/test_regress/t/t_benchmarksim.pl b/test_regress/t/t_benchmarksim.pl new file mode 100755 index 000000000..d8a8bb18f --- /dev/null +++ b/test_regress/t/t_benchmarksim.pl @@ -0,0 +1,52 @@ +#!/usr/bin/env perl +if (!$::Driver) { use FindBin; exec("$FindBin::Bin/bootstrap.pl", @ARGV, $0); die; } +# DESCRIPTION: Verilator: Verilog Test driver/expect definition +# +# Copyright 2003 by Wilson Snyder. This program is free software; you +# can redistribute it and/or modify it under the terms of either the GNU +# Lesser General Public License Version 3 or the Perl Artistic License +# Version 2.0. +# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 + +scenarios(vlt => 1); + +# Use any top file +top_filename("t/t_gen_alw.v"); + +init_benchmarksim(); + +# As an example, compile and simulate the top file with varying optimization level +my @l_opt = (1,2,3); + +foreach my $l_opt (@l_opt) { + compile( + benchmarksim => 1, + v_flags2 => ["-O$l_opt"] + ); + + execute( + check_finished => 1, + ); +} + +my $fh = IO::File->new("<".benchmarksim_filename()) or error("Benchmark data file not found"); +my $lines = 0; +while (defined(my $line = $fh->getline)) { + if ($line =~ /^#/) { next; } + if ($lines == 0) { + error("Expected header but found $line") if $line ne "evals, time[s]\n"; + } else { + my @data = grep {$_ != ""} ($line =~ /(\d*\.?\d*)/g); + error("Expected 2 tokens on line ".$lines." but got ".scalar(@data)) if scalar(@data) != 2; + my $cycles = $data[0]; + my $time = $data[1]; + error("Invalid data on line ".$lines) if $cycles <= 0.0 || $time <= 0.0; + } + $lines += 1; +} +my $n_lines_expected = scalar(@l_opt) + 1; + +error("Expected ".$n_lines_expected." lines but found ".$lines) if int($lines) != int($n_lines_expected); + +1; +ok(1);