CI: exploit concurrency (#2687)

Co-authored-by: Wilson Snyder <wsnyder@wsnyder.org>
2024-12-29 10:47:34 +00:00 · 2020-12-23 17:53:05 +00:00 · 2020-12-23 17:53:05 +00:00 · 6e3de7bfd1
commit 6e3de7bfd1
parent 1124829500
7 changed files with 164 additions and 84 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -11,67 +11,127 @@ on:
  schedule:
  - cron: '0 0 * * 0' # weekly

+env:
+  CI_OS_NAME: linux
+  CI_COMMIT: ${{ github.sha }}
+  CCACHE_COMPRESS: 1
+  CCACHE_DIR: ${{ github.workspace }}/.ccache
+  CCACHE_MAXSIZE: 2Gi  # 2GiB for clang and gcc, 4GiB in total
+
 jobs:

-  vlt:
+
+  Matrix:
+    runs-on: ubuntu-latest
+    name: Generate Build matrix
+    outputs:
+      matrix: ${{ steps.generate.outputs.matrix }}
+    steps:
+    - uses: actions/checkout@v2
+    - id: generate
+      name: Run 'generate_matrix.sh'
+      run: |
+        if [ '${{ github.event_name}}' = 'pull_request' ]; then
+          matrix='[ "ubuntu-20.04" ]'
+        else
+          matrix='[ "ubuntu-16.04", "ubuntu-18.04", "ubuntu-20.04" ]'
+        fi
+        echo "::set-output name=matrix::$matrix"
+
+
+  Build:
+    needs: Matrix
    strategy:
      fail-fast: false
      matrix:
-        include:
+        os: ${{ fromJson(needs.Matrix.outputs.matrix) }}
+        debug: [ opt, dbg ]
+        compiler:
          - { cc: clang, cxx: clang++ }
          - { cc: gcc,   cxx: g++     }
-    runs-on: ubuntu-20.04
-    name: Ubuntu 20.04 | ${{ matrix.cc }} | build-test
+    runs-on: ${{ matrix.os }}
+    name: Build | ${{ matrix.os }} | ${{ matrix.compiler.cc }} | ${{ matrix.debug }}
    env:
-      CI_OS_NAME: linux
-      CI_RUNS_ON: ubuntu-20.04
-      CI_COMMIT: ${{ github.sha }}
-      CCACHE_COMPRESS: 1
-      CCACHE_DIR: ${{ github.workspace }}/.ccache
-      CCACHE_MAXSIZE: 2Gi  # 2GiB for clang and gcc, 4GiB in total
-      CC: ${{ matrix.cc }}
-      CXX: ${{ matrix.cxx }}
+      CI_BUILD_STAGE_NAME: build
+      CI_RUNS_ON: ${{ matrix.os }}
+      CI_MAKE_SRC_TARGET: ${{ matrix.debug }}
+      CC: ${{ matrix.compiler.cc }}
+      CXX: ${{ matrix.compiler.cxx }}
+      CACHE_KEY: ${{ matrix.os }}-${{ matrix.compiler.cc }}
    steps:

-      - name: Checkout
-        uses: actions/checkout@v2
+    - name: Checkout
+      uses: actions/checkout@v2

-      - name: Cache
-        uses: actions/cache@v2
-        env:
-          cache-name: ccache
-        with:
-          path: ${{ github.workspace }}/.ccache
-          key: ubuntu-20.04-${{ matrix.cc }}-${{ env.cache-name }}-${{ github.sha }}
-          restore-keys: |
-            ubuntu-20.04-${{ matrix.cc }}-${{ env.cache-name }}
+    - name: Cache
+      uses: actions/cache@v2
+      env:
+        cache-name: ccache
+      with:
+        path: ${{ github.workspace }}/.ccache
+        key: ${{ env.CACHE_KEY }}-${{ env.cache-name }}-${{ github.sha }}
+        restore-keys: ${{ env.CACHE_KEY }}-${{ env.cache-name }}

-      - name: Install packages for build
-        env:
-          CI_BUILD_STAGE_NAME: build
-        run: bash ci/ci-install.bash
+    - name: Install packages for build
+      run: ./ci/ci-install.bash

-      - name: CCACHE maintenance
-        run: mkdir -p $CCACHE_DIR && bash ci/ci-ccache-maint.bash
+    - name: Build
+      run: |
+        ./ci/ci-script.bash
+        tar cvzf verilator-${{ matrix.os}}-${CI_COMMIT}-${{ matrix.compiler.cc }}-${{ matrix.debug }}.tgz ./bin

-      - name: Build
-        env:
-          CI_BUILD_STAGE_NAME: build
-        run: bash ci/ci-script.bash
+    - uses: actions/upload-artifact@v2
+      with:
+        path: verilator-${{ matrix.os}}-${{ env.CI_COMMIT }}-${{ matrix.compiler.cc }}-${{ matrix.debug }}.tgz

-      - name: Install packages for tests
-        env:
-          CI_BUILD_STAGE_NAME: test
-        run: bash ci/ci-install.bash

-      - name: Test
-        env:
-          CI_BUILD_STAGE_NAME: test
-        run: |
-          for item in dist-vlt-0 dist-vlt-1 vltmt-0 vltmt-1; do
-            echo "::group::${item}"
+  Test:
+    needs: [ Matrix, Build ]
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        os: ${{ fromJson(needs.Matrix.outputs.matrix) }}
+        compiler:
+          - { cc: clang, cxx: clang++ }
+          - { cc: gcc,   cxx: g++     }
+        suite:
+          - dist-vlt-0
+          - dist-vlt-1
+          - vltmt-0
+          - vltmt-1
+    runs-on: ${{ matrix.os }}
+    name: Test | ${{ matrix.os }} | ${{ matrix.compiler.cc }} | ${{ matrix.suite }}
+    env:
+      CI_BUILD_STAGE_NAME: test
+      CI_RUNS_ON: ${{ matrix.os }}
+      CC: ${{ matrix.compiler.cc }}
+      CXX: ${{ matrix.compiler.cxx }}
+      CACHE_KEY: ${{ matrix.os }}-${{ matrix.compiler.cc }}
+    steps:

-            TESTS="${item}" bash ci/ci-script.bash
+    - name: Checkout
+      uses: actions/checkout@v2

-            echo '::endgroup::'
-          done
+    - name: Cache
+      uses: actions/cache@v2
+      env:
+        cache-name: ccache
+      with:
+        path: ${{ github.workspace }}/.ccache
+        key: ${{ env.CACHE_KEY }}-${{ env.cache-name }}-${{ github.sha }}
+        restore-keys: ${{ env.CACHE_KEY }}-${{ env.cache-name }}
+
+    - uses: actions/download-artifact@v2
+
+    - name: Install Verilator and test dependencies
+      run: |
+        tar xvzf artifact/verilator-${{ matrix.os}}-${CI_COMMIT}-${{ matrix.compiler.cc }}-opt.tgz
+        tar xvzf artifact/verilator-${{ matrix.os}}-${CI_COMMIT}-${{ matrix.compiler.cc }}-dbg.tgz
+        rm -rf artifact
+        ./ci/ci-install.bash
+
+    - name: Test
+      env:
+        TESTS: ${{ matrix.suite }}
+      run: ./ci/ci-script.bash
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@ -15,27 +15,47 @@ env:
  CCACHE_DIR: ${{ github.workspace }}/.ccache
  CCACHE_MAXSIZE: 2Gi  # 2GiB for clang and gcc, 4GiB in total
  COVERAGE: 1
-  CI_BUILD_STAGE_NAME: test

 jobs:

-  buildjob:
+
+  Build:
    runs-on: ubuntu-20.04
+    env:
+      CI_BUILD_STAGE_NAME: build
    steps:
+
    - uses: actions/checkout@v2
+
    - uses: actions/cache@v2
      env:
        cache-name: ccache
      with:
        path: ${{ github.workspace }}/.ccache
        key: coverage-${{ env.cache-name }}-${{ github.sha }}
-        restore-keys: |
-          coverage-${{ env.cache-name }}
-    - run: bash -c "CI_BUILD_STAGE_NAME=build ci/ci-install-build.bash"
+        restore-keys: coverage-${{ env.cache-name }}
+
+    - name: Install dependencies for build
+      env:
+        CI_BUILD_STAGE_NAME: build
+      run: ./ci/ci-install.bash
+
+    - name: Build Verilator
+      run: |
+        ./ci/ci-script.bash
+
+        mkdir tarball/
+        cp -vr bin tarball/
+        cp -vr src/obj*/*.gcno tarball/
+        tar cvzf verilator-${CI_COMMIT}.tgz -C ./tarball .
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: verilator-${{ env.CI_COMMIT }}.tgz


-  test:
-    needs: buildjob
+  Test:
+    needs: Build
    strategy:
      fail-fast: false
      matrix:
@ -51,15 +71,28 @@ jobs:
        - { test: dist, num: '' }
    name: test-${{ matrix.test }}${{ matrix.num }}
    runs-on: ubuntu-20.04
+    env:
+      CI_BUILD_STAGE_NAME: test
    steps:
+
    - uses: actions/checkout@v2
+
    - uses: actions/cache@v2
      env:
        cache-name: ccache
      with:
        path: ${{ github.workspace }}/.ccache
        key: coverage-${{ env.cache-name }}-${{ github.sha }}
-        restore-keys: |
-          coverage-${{ env.cache-name }}
-    - run: bash -c "CI_BUILD_STAGE_NAME=build ci/ci-install-build.bash"
-    - run: bash -c "TESTS=coverage-${{ matrix.test }}${{ matrix.num }} ci/ci-script.bash"
+        restore-keys: coverage-${{ env.cache-name }}
+
+    - uses: actions/download-artifact@v2
+
+    - name: Install Verilator and test dependencies
+      run: |
+        tar xvzf artifact/verilator-${CI_COMMIT}.tgz
+        ./ci/ci-install.bash
+
+    - name: Test
+      env:
+        TESTS: coverage-${{ matrix.test }}${{ matrix.num }}
+      run: ./ci/ci-script.bash
--- a/.gitignore
+++ b/.gitignore
@ -16,6 +16,7 @@
 *.pdf
 /Makefile
 /.ccache
+/artifact/
 README
 TAGS
 autom4te.cache
--- a/Makefile.in
+++ b/Makefile.in
@ -219,7 +219,7 @@ all_nomsg: verilator_exe $(VL_INST_MAN_FILES)
 verilator_exe verilator_bin verilator_bin_dbg verilator_coverage_bin_dbg:
 	@echo ------------------------------------------------------------
 	@echo "making verilator in src"
-	$(MAKE) -C src $(OBJCACHE_JOBS)
+	$(MAKE) -C src $(OBJCACHE_JOBS) $(CI_MAKE_SRC_TARGET)

 .PHONY:msg_test
 msg_test: all_nomsg
--- a/ci/ci-install-build.bash
+++ b/ci/ci-install-build.bash
@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# DESCRIPTION: Verilator: CI main job script
-#
-# Copyright 2020 by Geza Lore. This program is free software; you
-# can redistribute it and/or modify it under the terms of either the GNU
-# Lesser General Public License Version 3 or the Perl Artistic License
-# Version 2.0.
-#
-# SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0
-
-echo "::group::build's ci-install"
-export CI_BUILD_STAGE_NAME=build
-bash ci/ci-install.bash
-echo "::endgroup"
-
-echo "::group::build's ci-script"
-mkdir -p $CCACHE_DIR && bash ci/ci-ccache-maint.bash
-bash ci/ci-script.bash
-echo "::endgroup"
-
-echo "::group::test's ci-install"
-export CI_BUILD_STAGE_NAME=test
-bash ci/ci-install.bash
-echo "::endgroup"
--- a/ci/ci-install.bash
+++ b/ci/ci-install.bash
@ -17,6 +17,8 @@
 set -e
 set -x

+cd $(dirname "$0")/..
+
 fatal() {
  echo "ERROR: $(basename "$0"): $1" >&2; exit 1;
 }
@ -66,6 +68,8 @@ if [ "$CI_BUILD_STAGE_NAME" = "build" ]; then
  else
    fatal "Unknown os: '$CI_OS_NAME'"
  fi
+
+  mkdir -p "$CCACHE_DIR" && ./ci/ci-ccache-maint.bash
 elif [ "$CI_BUILD_STAGE_NAME" = "test" ]; then
  ##############################################################################
  # Dependencies of jobs in the 'test' stage, i.e.: packages required to
@ -96,6 +100,9 @@ elif [ "$CI_BUILD_STAGE_NAME" = "test" ]; then
  fi
  yes yes | sudo cpan -M $CI_CPAN_REPO -fi Unix::Processors Parallel::Forker
  install-vcddiff
+
+  autoconf
+  ./configure --enable-longtests --enable-ccwarn
 else
  ##############################################################################
  # Unknown build stage
--- a/ci/ci-script.bash
+++ b/ci/ci-script.bash
@ -79,13 +79,16 @@ elif [ "$CI_BUILD_STAGE_NAME" = "test" ]; then
    export VERILATOR_TEST_NO_GPROF=1 # gprof is a bit different on FreeBSD, disable
  fi

+  # Run sanitize on Ubuntu 20.04 only
+  [ "$CI_RUNS_ON" = 'ubuntu-20.04' ] && sanitize='--sanitize' || sanitize=''
+
  # Run the specified test
  case $TESTS in
    dist-vlt-0)
-      "$MAKE" -C test_regress SCENARIOS="--dist --vlt --sanitize" DRIVER_HASHSET=--hashset=0/2
+      "$MAKE" -C test_regress SCENARIOS="--dist --vlt $sanitize" DRIVER_HASHSET=--hashset=0/2
      ;;
    dist-vlt-1)
-      "$MAKE" -C test_regress SCENARIOS="--dist --vlt --sanitize" DRIVER_HASHSET=--hashset=1/2
+      "$MAKE" -C test_regress SCENARIOS="--dist --vlt $sanitize" DRIVER_HASHSET=--hashset=1/2
      ;;
    vltmt-0)
      "$MAKE" -C test_regress SCENARIOS=--vltmt DRIVER_HASHSET=--hashset=0/2