From 6e3de7bfd14953ec0f4b34b9d33594e1c078bc21 Mon Sep 17 00:00:00 2001 From: Unai Martinez-Corral <38422348+umarcor@users.noreply.github.com> Date: Wed, 23 Dec 2020 17:53:05 +0000 Subject: [PATCH] CI: exploit concurrency (#2687) Co-authored-by: Wilson Snyder --- .github/workflows/build.yml | 152 +++++++++++++++++++++++---------- .github/workflows/coverage.yml | 55 +++++++++--- .gitignore | 1 + Makefile.in | 2 +- ci/ci-install-build.bash | 24 ------ ci/ci-install.bash | 7 ++ ci/ci-script.bash | 7 +- 7 files changed, 164 insertions(+), 84 deletions(-) delete mode 100755 ci/ci-install-build.bash diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cd55813f5..98ba13ed7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,67 +11,127 @@ on: schedule: - cron: '0 0 * * 0' # weekly +env: + CI_OS_NAME: linux + CI_COMMIT: ${{ github.sha }} + CCACHE_COMPRESS: 1 + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_MAXSIZE: 2Gi # 2GiB for clang and gcc, 4GiB in total + jobs: - vlt: + + Matrix: + runs-on: ubuntu-latest + name: Generate Build matrix + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + steps: + - uses: actions/checkout@v2 + - id: generate + name: Run 'generate_matrix.sh' + run: | + if [ '${{ github.event_name}}' = 'pull_request' ]; then + matrix='[ "ubuntu-20.04" ]' + else + matrix='[ "ubuntu-16.04", "ubuntu-18.04", "ubuntu-20.04" ]' + fi + echo "::set-output name=matrix::$matrix" + + + Build: + needs: Matrix strategy: fail-fast: false matrix: - include: + os: ${{ fromJson(needs.Matrix.outputs.matrix) }} + debug: [ opt, dbg ] + compiler: - { cc: clang, cxx: clang++ } - { cc: gcc, cxx: g++ } - runs-on: ubuntu-20.04 - name: Ubuntu 20.04 | ${{ matrix.cc }} | build-test + runs-on: ${{ matrix.os }} + name: Build | ${{ matrix.os }} | ${{ matrix.compiler.cc }} | ${{ matrix.debug }} env: - CI_OS_NAME: linux - CI_RUNS_ON: ubuntu-20.04 - CI_COMMIT: ${{ github.sha }} - CCACHE_COMPRESS: 1 - CCACHE_DIR: ${{ github.workspace }}/.ccache - CCACHE_MAXSIZE: 2Gi # 2GiB for clang and gcc, 4GiB in total - CC: ${{ matrix.cc }} - CXX: ${{ matrix.cxx }} + CI_BUILD_STAGE_NAME: build + CI_RUNS_ON: ${{ matrix.os }} + CI_MAKE_SRC_TARGET: ${{ matrix.debug }} + CC: ${{ matrix.compiler.cc }} + CXX: ${{ matrix.compiler.cxx }} + CACHE_KEY: ${{ matrix.os }}-${{ matrix.compiler.cc }} steps: - - name: Checkout - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 - - name: Cache - uses: actions/cache@v2 - env: - cache-name: ccache - with: - path: ${{ github.workspace }}/.ccache - key: ubuntu-20.04-${{ matrix.cc }}-${{ env.cache-name }}-${{ github.sha }} - restore-keys: | - ubuntu-20.04-${{ matrix.cc }}-${{ env.cache-name }} + - name: Cache + uses: actions/cache@v2 + env: + cache-name: ccache + with: + path: ${{ github.workspace }}/.ccache + key: ${{ env.CACHE_KEY }}-${{ env.cache-name }}-${{ github.sha }} + restore-keys: ${{ env.CACHE_KEY }}-${{ env.cache-name }} - - name: Install packages for build - env: - CI_BUILD_STAGE_NAME: build - run: bash ci/ci-install.bash + - name: Install packages for build + run: ./ci/ci-install.bash - - name: CCACHE maintenance - run: mkdir -p $CCACHE_DIR && bash ci/ci-ccache-maint.bash + - name: Build + run: | + ./ci/ci-script.bash + tar cvzf verilator-${{ matrix.os}}-${CI_COMMIT}-${{ matrix.compiler.cc }}-${{ matrix.debug }}.tgz ./bin - - name: Build - env: - CI_BUILD_STAGE_NAME: build - run: bash ci/ci-script.bash + - uses: actions/upload-artifact@v2 + with: + path: verilator-${{ matrix.os}}-${{ env.CI_COMMIT }}-${{ matrix.compiler.cc }}-${{ matrix.debug }}.tgz - - name: Install packages for tests - env: - CI_BUILD_STAGE_NAME: test - run: bash ci/ci-install.bash - - name: Test - env: - CI_BUILD_STAGE_NAME: test - run: | - for item in dist-vlt-0 dist-vlt-1 vltmt-0 vltmt-1; do - echo "::group::${item}" + Test: + needs: [ Matrix, Build ] + strategy: + fail-fast: false + max-parallel: 8 + matrix: + os: ${{ fromJson(needs.Matrix.outputs.matrix) }} + compiler: + - { cc: clang, cxx: clang++ } + - { cc: gcc, cxx: g++ } + suite: + - dist-vlt-0 + - dist-vlt-1 + - vltmt-0 + - vltmt-1 + runs-on: ${{ matrix.os }} + name: Test | ${{ matrix.os }} | ${{ matrix.compiler.cc }} | ${{ matrix.suite }} + env: + CI_BUILD_STAGE_NAME: test + CI_RUNS_ON: ${{ matrix.os }} + CC: ${{ matrix.compiler.cc }} + CXX: ${{ matrix.compiler.cxx }} + CACHE_KEY: ${{ matrix.os }}-${{ matrix.compiler.cc }} + steps: - TESTS="${item}" bash ci/ci-script.bash + - name: Checkout + uses: actions/checkout@v2 - echo '::endgroup::' - done + - name: Cache + uses: actions/cache@v2 + env: + cache-name: ccache + with: + path: ${{ github.workspace }}/.ccache + key: ${{ env.CACHE_KEY }}-${{ env.cache-name }}-${{ github.sha }} + restore-keys: ${{ env.CACHE_KEY }}-${{ env.cache-name }} + + - uses: actions/download-artifact@v2 + + - name: Install Verilator and test dependencies + run: | + tar xvzf artifact/verilator-${{ matrix.os}}-${CI_COMMIT}-${{ matrix.compiler.cc }}-opt.tgz + tar xvzf artifact/verilator-${{ matrix.os}}-${CI_COMMIT}-${{ matrix.compiler.cc }}-dbg.tgz + rm -rf artifact + ./ci/ci-install.bash + + - name: Test + env: + TESTS: ${{ matrix.suite }} + run: ./ci/ci-script.bash diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 905757bbf..00623317b 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -15,27 +15,47 @@ env: CCACHE_DIR: ${{ github.workspace }}/.ccache CCACHE_MAXSIZE: 2Gi # 2GiB for clang and gcc, 4GiB in total COVERAGE: 1 - CI_BUILD_STAGE_NAME: test jobs: - buildjob: + + Build: runs-on: ubuntu-20.04 + env: + CI_BUILD_STAGE_NAME: build steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 env: cache-name: ccache with: path: ${{ github.workspace }}/.ccache key: coverage-${{ env.cache-name }}-${{ github.sha }} - restore-keys: | - coverage-${{ env.cache-name }} - - run: bash -c "CI_BUILD_STAGE_NAME=build ci/ci-install-build.bash" + restore-keys: coverage-${{ env.cache-name }} + + - name: Install dependencies for build + env: + CI_BUILD_STAGE_NAME: build + run: ./ci/ci-install.bash + + - name: Build Verilator + run: | + ./ci/ci-script.bash + + mkdir tarball/ + cp -vr bin tarball/ + cp -vr src/obj*/*.gcno tarball/ + tar cvzf verilator-${CI_COMMIT}.tgz -C ./tarball . + + - uses: actions/upload-artifact@v2 + with: + path: verilator-${{ env.CI_COMMIT }}.tgz - test: - needs: buildjob + Test: + needs: Build strategy: fail-fast: false matrix: @@ -51,15 +71,28 @@ jobs: - { test: dist, num: '' } name: test-${{ matrix.test }}${{ matrix.num }} runs-on: ubuntu-20.04 + env: + CI_BUILD_STAGE_NAME: test steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 env: cache-name: ccache with: path: ${{ github.workspace }}/.ccache key: coverage-${{ env.cache-name }}-${{ github.sha }} - restore-keys: | - coverage-${{ env.cache-name }} - - run: bash -c "CI_BUILD_STAGE_NAME=build ci/ci-install-build.bash" - - run: bash -c "TESTS=coverage-${{ matrix.test }}${{ matrix.num }} ci/ci-script.bash" + restore-keys: coverage-${{ env.cache-name }} + + - uses: actions/download-artifact@v2 + + - name: Install Verilator and test dependencies + run: | + tar xvzf artifact/verilator-${CI_COMMIT}.tgz + ./ci/ci-install.bash + + - name: Test + env: + TESTS: coverage-${{ matrix.test }}${{ matrix.num }} + run: ./ci/ci-script.bash diff --git a/.gitignore b/.gitignore index 7a6a696dd..8721ca5a8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ *.pdf /Makefile /.ccache +/artifact/ README TAGS autom4te.cache diff --git a/Makefile.in b/Makefile.in index 0c033bab6..85ef99c93 100644 --- a/Makefile.in +++ b/Makefile.in @@ -219,7 +219,7 @@ all_nomsg: verilator_exe $(VL_INST_MAN_FILES) verilator_exe verilator_bin verilator_bin_dbg verilator_coverage_bin_dbg: @echo ------------------------------------------------------------ @echo "making verilator in src" - $(MAKE) -C src $(OBJCACHE_JOBS) + $(MAKE) -C src $(OBJCACHE_JOBS) $(CI_MAKE_SRC_TARGET) .PHONY:msg_test msg_test: all_nomsg diff --git a/ci/ci-install-build.bash b/ci/ci-install-build.bash deleted file mode 100755 index 39fbfc936..000000000 --- a/ci/ci-install-build.bash +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# DESCRIPTION: Verilator: CI main job script -# -# Copyright 2020 by Geza Lore. This program is free software; you -# can redistribute it and/or modify it under the terms of either the GNU -# Lesser General Public License Version 3 or the Perl Artistic License -# Version 2.0. -# -# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 - -echo "::group::build's ci-install" -export CI_BUILD_STAGE_NAME=build -bash ci/ci-install.bash -echo "::endgroup" - -echo "::group::build's ci-script" -mkdir -p $CCACHE_DIR && bash ci/ci-ccache-maint.bash -bash ci/ci-script.bash -echo "::endgroup" - -echo "::group::test's ci-install" -export CI_BUILD_STAGE_NAME=test -bash ci/ci-install.bash -echo "::endgroup" diff --git a/ci/ci-install.bash b/ci/ci-install.bash index 302b02cb9..de46dfdda 100755 --- a/ci/ci-install.bash +++ b/ci/ci-install.bash @@ -17,6 +17,8 @@ set -e set -x +cd $(dirname "$0")/.. + fatal() { echo "ERROR: $(basename "$0"): $1" >&2; exit 1; } @@ -66,6 +68,8 @@ if [ "$CI_BUILD_STAGE_NAME" = "build" ]; then else fatal "Unknown os: '$CI_OS_NAME'" fi + + mkdir -p "$CCACHE_DIR" && ./ci/ci-ccache-maint.bash elif [ "$CI_BUILD_STAGE_NAME" = "test" ]; then ############################################################################## # Dependencies of jobs in the 'test' stage, i.e.: packages required to @@ -96,6 +100,9 @@ elif [ "$CI_BUILD_STAGE_NAME" = "test" ]; then fi yes yes | sudo cpan -M $CI_CPAN_REPO -fi Unix::Processors Parallel::Forker install-vcddiff + + autoconf + ./configure --enable-longtests --enable-ccwarn else ############################################################################## # Unknown build stage diff --git a/ci/ci-script.bash b/ci/ci-script.bash index c75401b93..bcfc24285 100755 --- a/ci/ci-script.bash +++ b/ci/ci-script.bash @@ -79,13 +79,16 @@ elif [ "$CI_BUILD_STAGE_NAME" = "test" ]; then export VERILATOR_TEST_NO_GPROF=1 # gprof is a bit different on FreeBSD, disable fi + # Run sanitize on Ubuntu 20.04 only + [ "$CI_RUNS_ON" = 'ubuntu-20.04' ] && sanitize='--sanitize' || sanitize='' + # Run the specified test case $TESTS in dist-vlt-0) - "$MAKE" -C test_regress SCENARIOS="--dist --vlt --sanitize" DRIVER_HASHSET=--hashset=0/2 + "$MAKE" -C test_regress SCENARIOS="--dist --vlt $sanitize" DRIVER_HASHSET=--hashset=0/2 ;; dist-vlt-1) - "$MAKE" -C test_regress SCENARIOS="--dist --vlt --sanitize" DRIVER_HASHSET=--hashset=1/2 + "$MAKE" -C test_regress SCENARIOS="--dist --vlt $sanitize" DRIVER_HASHSET=--hashset=1/2 ;; vltmt-0) "$MAKE" -C test_regress SCENARIOS=--vltmt DRIVER_HASHSET=--hashset=0/2