From 93d50c44993cf879375a1ad93a1297a3fab0eaaf Mon Sep 17 00:00:00 2001
From: Kamil Rakoczy <krakoczy@antmicro.com>
Date: Fri, 10 Feb 2023 04:15:37 +0100
Subject: [PATCH] Internals: Add mutex to V3Error (#3680)

---
 include/verilated.h                   |  27 ++-
 src/V3Ast.cpp                         |  10 +-
 src/V3Ast.h                           |  13 +-
 src/V3Dfg.h                           |  15 +-
 src/V3Error.cpp                       | 233 ++++++++++---------
 src/V3Error.h                         | 316 ++++++++++++++++++++------
 src/V3FileLine.cpp                    |  54 +++--
 src/V3FileLine.h                      |  21 +-
 src/V3Graph.cpp                       |   7 +-
 src/V3Graph.h                         |   4 +-
 src/V3LinkLevel.cpp                   |  18 +-
 src/V3Number.cpp                      |   7 +-
 src/V3Number.h                        |   5 +-
 src/V3Options.cpp                     |  12 +-
 src/V3PreProc.cpp                     |   4 +-
 src/V3SchedAcyclic.cpp                |  12 +-
 src/V3SymTable.h                      |   2 +-
 src/V3Width.cpp                       |  10 +-
 test_regress/t/t_a5_attributes_src.pl |   2 +-
 19 files changed, 505 insertions(+), 267 deletions(-)

diff --git a/include/verilated.h b/include/verilated.h
index 0b4592b27..db8cf9f63 100644
--- a/include/verilated.h
+++ b/include/verilated.h
@@ -47,6 +47,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <deque>
+#include <functional>
 #include <map>
 #include <memory>
 #include <set>
@@ -175,6 +176,19 @@ public:
     void unlock() VL_RELEASE() VL_MT_SAFE { m_mutex.unlock(); }
     /// Try to acquire mutex.  Returns true on success, and false on failure.
     bool try_lock() VL_TRY_ACQUIRE(true) VL_MT_SAFE { return m_mutex.try_lock(); }
+    /// Acquire/lock mutex and check for stop request
+    /// It tries to lock the mutex and if it fails, it check if stop request was send.
+    /// It returns after locking mutex.
+    /// This function should be extracted to V3ThreadPool, but due to clang thread-safety
+    /// limitations it needs to be placed here.
+    void lockCheckStopRequest(std::function<void()> checkStopRequestFunction)
+        VL_ACQUIRE() VL_MT_SAFE {
+        while (true) {
+            checkStopRequestFunction();
+            if (m_mutex.try_lock()) return;
+            VL_CPU_RELAX();
+        }
+    }
 };
 
 /// Lock guard for mutex (ala std::unique_lock), wrapped to allow -fthread_safety checks
@@ -192,10 +206,19 @@ public:
     }
     /// Destruct and unlock the mutex
     ~VerilatedLockGuard() VL_RELEASE() { m_mutexr.unlock(); }
-    /// Unlock the mutex
-    void lock() VL_ACQUIRE() VL_MT_SAFE { m_mutexr.lock(); }
     /// Lock the mutex
+    void lock() VL_ACQUIRE() VL_MT_SAFE { m_mutexr.lock(); }
+    /// Unlock the mutex
     void unlock() VL_RELEASE() VL_MT_SAFE { m_mutexr.unlock(); }
+    /// Acquire/lock mutex and check for stop request.
+    /// It tries to lock the mutex and if it fails, it check if stop request was send.
+    /// It returns after locking mutex.
+    /// This function should be extracted to V3ThreadPool, but due to clang thread-safety
+    /// limitations it needs to be placed here.
+    void lockCheckStopRequest(std::function<void()> checkStopRequestFunction)
+        VL_ACQUIRE() VL_MT_SAFE {
+        m_mutexr.lockCheckStopRequest(checkStopRequestFunction);
+    }
 };
 
 // Internals: Remember the calling thread at construction time, and make
diff --git a/src/V3Ast.cpp b/src/V3Ast.cpp
index cf8b6b1bb..689c000d3 100644
--- a/src/V3Ast.cpp
+++ b/src/V3Ast.cpp
@@ -1257,7 +1257,7 @@ void AstNode::dumpTreeDotFile(const string& filename, bool append, bool doDump)
     }
 }
 
-void AstNode::v3errorEndFatal(std::ostringstream& str) const VL_MT_SAFE {
+void AstNode::v3errorEndFatal(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex) {
     v3errorEnd(str);
     assert(0);  // LCOV_EXCL_LINE
     VL_UNREACHABLE;
@@ -1287,9 +1287,9 @@ string AstNode::instanceStr() const {
 
     return "";
 }
-void AstNode::v3errorEnd(std::ostringstream& str) const {
+void AstNode::v3errorEnd(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex) {
     if (!m_fileline) {
-        V3Error::v3errorEnd(str, instanceStr());
+        V3Error::s().v3errorEnd(str, instanceStr());
     } else {
         std::ostringstream nsstr;
         nsstr << str.str();
@@ -1302,8 +1302,8 @@ void AstNode::v3errorEnd(std::ostringstream& str) const {
         // Don't look for instance name when warning is disabled.
         // In case of large number of warnings, this can
         // take significant amount of time
-        m_fileline->v3errorEnd(nsstr,
-                               m_fileline->warnIsOff(V3Error::errorCode()) ? "" : instanceStr());
+        m_fileline->v3errorEnd(
+            nsstr, m_fileline->warnIsOff(V3Error::s().errorCode()) ? "" : instanceStr());
     }
 }
 
diff --git a/src/V3Ast.h b/src/V3Ast.h
index eb8e03d19..c4f9e6dd4 100644
--- a/src/V3Ast.h
+++ b/src/V3Ast.h
@@ -1853,12 +1853,15 @@ public:
     static AstBasicDType* findInsertSameDType(AstBasicDType* nodep);
 
     // METHODS - dump and error
-    void v3errorEnd(std::ostringstream& str) const VL_MT_SAFE;
-    void v3errorEndFatal(std::ostringstream& str) const VL_ATTR_NORETURN VL_MT_SAFE;
-    string warnContextPrimary() const { return fileline()->warnContextPrimary(); }
+    void v3errorEnd(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex);
+    void v3errorEndFatal(std::ostringstream& str) const VL_ATTR_NORETURN
+        VL_REQUIRES(V3Error::s().m_mutex);
+    string warnContextPrimary() const VL_REQUIRES(V3Error::s().m_mutex) {
+        return fileline()->warnContextPrimary();
+    }
     string warnContextSecondary() const { return fileline()->warnContextSecondary(); }
-    string warnMore() const { return fileline()->warnMore(); }
-    string warnOther() const { return fileline()->warnOther(); }
+    string warnMore() const VL_REQUIRES(V3Error::s().m_mutex) { return fileline()->warnMore(); }
+    string warnOther() const VL_REQUIRES(V3Error::s().m_mutex) { return fileline()->warnOther(); }
 
     virtual void dump(std::ostream& str = std::cout) const;
     static void dumpGdb(const AstNode* nodep);  // For GDB only
diff --git a/src/V3Dfg.h b/src/V3Dfg.h
index ff6908d43..88c868192 100644
--- a/src/V3Dfg.h
+++ b/src/V3Dfg.h
@@ -503,14 +503,19 @@ public:
     inline bool inlined() const;
 
     // Methods that allow DfgVertex to participate in error reporting/messaging
-    void v3errorEnd(std::ostringstream& str) const { m_filelinep->v3errorEnd(str); }
-    void v3errorEndFatal(std::ostringstream& str) const VL_ATTR_NORETURN {
+    void v3errorEnd(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex) {
+        m_filelinep->v3errorEnd(str);
+    }
+    void v3errorEndFatal(std::ostringstream& str) const VL_ATTR_NORETURN
+        VL_REQUIRES(V3Error::s().m_mutex) {
         m_filelinep->v3errorEndFatal(str);
     }
-    string warnContextPrimary() const { return fileline()->warnContextPrimary(); }
+    string warnContextPrimary() const VL_REQUIRES(V3Error::s().m_mutex) {
+        return fileline()->warnContextPrimary();
+    }
     string warnContextSecondary() const { return fileline()->warnContextSecondary(); }
-    string warnMore() const { return fileline()->warnMore(); }
-    string warnOther() const { return fileline()->warnOther(); }
+    string warnMore() const VL_REQUIRES(V3Error::s().m_mutex) { return fileline()->warnMore(); }
+    string warnOther() const VL_REQUIRES(V3Error::s().m_mutex) { return fileline()->warnOther(); }
 
 private:
     // For internal use only.
diff --git a/src/V3Error.cpp b/src/V3Error.cpp
index 0a3515219..2f1a01f20 100644
--- a/src/V3Error.cpp
+++ b/src/V3Error.cpp
@@ -25,24 +25,6 @@ VL_DEFINE_DEBUG_FUNCTIONS;
 // clang-format on
 
 //======================================================================
-// Statics
-
-int V3Error::s_errCount = 0;
-int V3Error::s_warnCount = 0;
-int V3Error::s_debugDefault = 0;
-int V3Error::s_errorLimit = V3Error::MAX_ERRORS;
-bool V3Error::s_warnFatal = true;
-int V3Error::s_tellManual = 0;
-std::ostringstream V3Error::s_errorStr;  // Error string being formed
-V3ErrorCode V3Error::s_errorCode = V3ErrorCode::EC_FATAL;
-bool V3Error::s_errorContexted = false;
-bool V3Error::s_errorSuppressed = false;
-std::array<bool, V3ErrorCode::_ENUM_MAX> V3Error::s_describedEachWarn;
-std::array<bool, V3ErrorCode::_ENUM_MAX> V3Error::s_pretendError;
-bool V3Error::s_describedWarnings = false;
-bool V3Error::s_describedWeb = false;
-V3Error::MessagesSet V3Error::s_messages;
-V3Error::ErrorExitCb V3Error::s_errorExitCb = nullptr;
 
 struct v3errorIniter {
     v3errorIniter() { V3Error::init(); }
@@ -65,51 +47,10 @@ V3ErrorCode::V3ErrorCode(const char* msgp) {
 }
 
 //######################################################################
-// V3Error class functions
+// V3ErrorGuarded class functions
+//
 
-void V3Error::init() {
-    for (int i = 0; i < V3ErrorCode::_ENUM_MAX; i++) {
-        s_describedEachWarn[i] = false;
-        s_pretendError[i] = V3ErrorCode{i}.pretendError();
-    }
-    if (VL_UNCOVERABLE(string(V3ErrorCode{V3ErrorCode::_ENUM_MAX}.ascii()) != " MAX")) {
-        v3fatalSrc("Enum table in V3ErrorCode::EC_ascii() is munged");
-    }
-}
-
-string V3Error::lineStr(const char* filename, int lineno) {
-    std::ostringstream out;
-    const char* const fnslashp = std::strrchr(filename, '/');
-    if (fnslashp) filename = fnslashp + 1;
-    out << filename << ":" << std::dec << lineno << ":";
-    const char* const spaces = "                    ";
-    size_t numsp = out.str().length();
-    if (numsp > 20) numsp = 20;
-    out << (spaces + numsp);
-    return out.str();
-}
-
-void V3Error::incErrors() {
-    s_errCount++;
-    if (errorCount() == errorLimit()) {  // Not >= as would otherwise recurse
-        v3fatalExit("Exiting due to too many errors encountered; --error-limit="  //
-                    << errorCount() << endl);
-    }
-}
-
-void V3Error::abortIfWarnings() {
-    const bool exwarn = warnFatal() && warnCount();
-    if (errorCount() && exwarn) {
-        v3fatalExit("Exiting due to " << std::dec << errorCount() << " error(s), "  //
-                                      << warnCount() << " warning(s)\n");
-    } else if (errorCount()) {
-        v3fatalExit("Exiting due to " << std::dec << errorCount() << " error(s)\n");
-    } else if (exwarn) {
-        v3fatalExit("Exiting due to " << std::dec << warnCount() << " warning(s)\n");
-    }
-}
-
-bool V3Error::isError(V3ErrorCode code, bool supp) {
+bool V3ErrorGuarded::isError(V3ErrorCode code, bool supp) VL_REQUIRES(m_mutex) {
     if (supp) {
         return false;
     } else if (code == V3ErrorCode::USERINFO) {
@@ -124,16 +65,16 @@ bool V3Error::isError(V3ErrorCode code, bool supp) {
         return true;
     } else if (code == V3ErrorCode::EC_ERROR) {
         return true;
-    } else if (code < V3ErrorCode::EC_FIRST_WARN || s_pretendError[code]) {
+    } else if (code < V3ErrorCode::EC_FIRST_WARN || pretendError(code)) {
         return true;
     } else {
         return false;
     }
 }
 
-string V3Error::msgPrefix() {
-    const V3ErrorCode code = s_errorCode;
-    const bool supp = s_errorSuppressed;
+string V3ErrorGuarded::msgPrefix() VL_REQUIRES(m_mutex) {
+    const V3ErrorCode code = m_errorCode;
+    const bool supp = m_errorSuppressed;
     if (supp) {
         return "-arning-suppressed: ";
     } else if (code == V3ErrorCode::USERINFO) {
@@ -155,10 +96,7 @@ string V3Error::msgPrefix() {
     }
 }
 
-//======================================================================
-// Abort/exit
-
-void V3Error::vlAbortOrExit() {
+void V3ErrorGuarded::vlAbortOrExit() VL_REQUIRES(m_mutex) {
     if (V3Error::debugDefault()) {
         std::cerr << msgPrefix() << "Aborting since under --debug" << endl;
         V3Error::vlAbort();
@@ -167,50 +105,53 @@ void V3Error::vlAbortOrExit() {
     }
 }
 
-void V3Error::vlAbort() {
-    VL_GCOV_DUMP();
-    std::abort();
-}
+string V3ErrorGuarded::warnMore() VL_REQUIRES(m_mutex) { return string(msgPrefix().size(), ' '); }
 
-//======================================================================
-// Global Functions
-
-void V3Error::suppressThisWarning() {
+void V3ErrorGuarded::suppressThisWarning() VL_REQUIRES(m_mutex) {
 #ifndef V3ERROR_NO_GLOBAL_
-    V3Stats::addStatSum(std::string{"Warnings, Suppressed "} + s_errorCode.ascii(), 1);
+    V3Stats::addStatSum(std::string{"Warnings, Suppressed "} + errorCode().ascii(), 1);
 #endif
-    s_errorSuppressed = true;
+    errorSuppressed(true);
 }
 
-string V3Error::warnMore() { return string(msgPrefix().size(), ' '); }
-
 // cppcheck-has-bug-suppress constParameter
-void V3Error::v3errorEnd(std::ostringstream& sstr, const string& extra) {
+void V3ErrorGuarded::v3errorEnd(std::ostringstream& sstr, const string& extra)
+    VL_REQUIRES(m_mutex) {
 #if defined(__COVERITY__) || defined(__cppcheck__)
-    if (s_errorCode == V3ErrorCode::EC_FATAL) __coverity_panic__(x);
+    if (m_errorCode == V3ErrorCode::EC_FATAL) __coverity_panic__(x);
 #endif
     // Skip suppressed messages
-    if (s_errorSuppressed
+    if (m_errorSuppressed
         // On debug, show only non default-off warning to prevent pages of warnings
-        && (!debug() || s_errorCode.defaultsOff()))
+        && (!debug() || m_errorCode.defaultsOff()))
         return;
     string msg = msgPrefix() + sstr.str();
-    if (s_errorSuppressed) {  // If suppressed print only first line to reduce verbosity
+    // If suppressed print only first line to reduce verbosity
+    if (m_errorSuppressed) {
         string::size_type pos;
         if ((pos = msg.find('\n')) != string::npos) {
             msg.erase(pos, msg.length() - pos);
             msg += "...";
         }
     }
+    string msg_additional;
+    {
+        string::size_type pos;
+        if ((pos = msg.find(V3Error::warnAdditionalInfo())) != string::npos) {
+            msg_additional = msg.substr(pos + V3Error::warnAdditionalInfo().size());
+            msg.erase(pos);
+        }
+    }
     // Trailing newline (generally not on messages) & remove dup newlines
     {
         msg += '\n';  // Trailing newlines generally not put on messages so add
         string::size_type pos;
         while ((pos = msg.find("\n\n")) != string::npos) msg.erase(pos + 1, 1);
+        while ((pos = msg_additional.find("\n\n")) != string::npos)
+            msg_additional.erase(pos + 1, 1);
     }
     // Suppress duplicate messages
-    if (s_messages.find(msg) != s_messages.end()) return;
-    s_messages.insert(msg);
+    if (!m_messages.insert(msg).second) return;
     if (!extra.empty()) {
         const string extraMsg = warnMore() + extra + "\n";
         const size_t pos = msg.find('\n');
@@ -219,44 +160,45 @@ void V3Error::v3errorEnd(std::ostringstream& sstr, const string& extra) {
     // Output
     if (
 #ifndef V3ERROR_NO_GLOBAL_
-        !(v3Global.opt.quietExit() && s_errorCode == V3ErrorCode::EC_FATALEXIT)
+        !(v3Global.opt.quietExit() && m_errorCode == V3ErrorCode::EC_FATALEXIT)
 #else
         true
 #endif
     ) {
         std::cerr << msg;
     }
-    if (!s_errorSuppressed
-        && !(s_errorCode == V3ErrorCode::EC_INFO || s_errorCode == V3ErrorCode::USERINFO)) {
-        const bool anError = isError(s_errorCode, s_errorSuppressed);
-        if (s_errorCode >= V3ErrorCode::EC_FIRST_NAMED && !s_describedWeb) {
-            s_describedWeb = true;
+    if (!m_errorSuppressed
+        && !(m_errorCode == V3ErrorCode::EC_INFO || m_errorCode == V3ErrorCode::USERINFO)) {
+        const bool anError = isError(m_errorCode, m_errorSuppressed);
+        if (m_errorCode >= V3ErrorCode::EC_FIRST_NAMED && !m_describedWeb) {
+            m_describedWeb = true;
             std::cerr << warnMore() << "... For " << (anError ? "error" : "warning")
-                      << " description see https://verilator.org/warn/" << s_errorCode.ascii()
+                      << " description see https://verilator.org/warn/" << m_errorCode.ascii()
                       << "?v=" << PACKAGE_VERSION_NUMBER_STRING << endl;
         }
-        if (!s_describedEachWarn[s_errorCode] && !s_pretendError[s_errorCode]) {
-            s_describedEachWarn[s_errorCode] = true;
-            if (s_errorCode >= V3ErrorCode::EC_FIRST_WARN && !s_describedWarnings) {
-                s_describedWarnings = true;
+        if (!m_describedEachWarn[m_errorCode] && !m_pretendError[m_errorCode]) {
+            m_describedEachWarn[m_errorCode] = true;
+            if (m_errorCode >= V3ErrorCode::EC_FIRST_WARN && !m_describedWarnings) {
+                m_describedWarnings = true;
                 std::cerr << warnMore() << "... Use \"/* verilator lint_off "
-                          << s_errorCode.ascii()
+                          << m_errorCode.ascii()
                           << " */\" and lint_on around source to disable this message." << endl;
             }
-            if (s_errorCode.dangerous()) {
+            if (m_errorCode.dangerous()) {
                 std::cerr << warnMore() << "*** See https://verilator.org/warn/"
-                          << s_errorCode.ascii() << " before disabling this,\n";
+                          << m_errorCode.ascii() << " before disabling this,\n";
                 std::cerr << warnMore() << "else you may end up with different sim results."
                           << endl;
             }
         }
+        if (!msg_additional.empty()) { std::cerr << msg_additional; }
         // If first warning is not the user's fault (internal/unsupported) then give the website
         // Not later warnings, as a internal may be caused by an earlier problem
-        if (s_tellManual == 0) {
-            if (s_errorCode.mentionManual() || sstr.str().find("Unsupported") != string::npos) {
-                s_tellManual = 1;
+        if (tellManual() == 0) {
+            if (m_errorCode.mentionManual() || sstr.str().find("Unsupported") != string::npos) {
+                tellManual(1);
             } else {
-                s_tellManual = 2;
+                tellManual(2);
             }
         }
         if (anError) {
@@ -264,26 +206,34 @@ void V3Error::v3errorEnd(std::ostringstream& sstr, const string& extra) {
         } else {
             incWarnings();
         }
-        if (s_errorCode == V3ErrorCode::EC_FATAL || s_errorCode == V3ErrorCode::EC_FATALEXIT
-            || s_errorCode == V3ErrorCode::EC_FATALSRC) {
+        if (m_errorCode == V3ErrorCode::EC_FATAL || m_errorCode == V3ErrorCode::EC_FATALEXIT
+            || m_errorCode == V3ErrorCode::EC_FATALSRC) {
             static bool inFatal = false;
             if (!inFatal) {
                 inFatal = true;
-                if (s_tellManual == 1) {
+                if (tellManual() == 1) {
                     std::cerr << warnMore()
                               << "... See the manual at https://verilator.org/verilator_doc.html "
                                  "for more assistance."
                               << endl;
-                    s_tellManual = 2;
+                    tellManual(2);
                 }
 #ifndef V3ERROR_NO_GLOBAL_
-                if (dumpTree()) {
-                    v3Global.rootp()->dumpTreeFile(v3Global.debugFilename("final.tree", 990));
-                }
-                if (debug()) {
-                    if (s_errorExitCb) s_errorExitCb();
-                    V3Stats::statsFinalAll(v3Global.rootp());
-                    V3Stats::statsReport();
+                if (dumpTree() || debug()) {
+                    V3ThreadPool::s().requestExclusiveAccess([&]() VL_REQUIRES(m_mutex) {
+                        if (dumpTree()) {
+                            v3Global.rootp()->dumpTreeFile(
+                                v3Global.debugFilename("final.tree", 990));
+                        }
+                        if (debug()) {
+                            execErrorExitCb();
+                            V3Stats::statsFinalAll(v3Global.rootp());
+                            V3Stats::statsReport();
+                        }
+                        // Abort in exclusive access to make sure other threads
+                        // don't change error code
+                        vlAbortOrExit();
+                    });
                 }
 #endif
             }
@@ -292,7 +242,52 @@ void V3Error::v3errorEnd(std::ostringstream& sstr, const string& extra) {
         } else if (anError) {
             // We don't dump tree on any error because a Visitor may be in middle of
             // a tree cleanup and cause a false broken problem.
-            if (s_errorExitCb) s_errorExitCb();
+            execErrorExitCb();
         }
     }
 }
+
+//######################################################################
+// V3Error class functions
+
+void V3Error::init() {
+    for (int i = 0; i < V3ErrorCode::_ENUM_MAX; i++) {
+        describedEachWarn(static_cast<V3ErrorCode>(i), false);
+        pretendError(static_cast<V3ErrorCode>(i), V3ErrorCode{i}.pretendError());
+    }
+    if (VL_UNCOVERABLE(string(V3ErrorCode{V3ErrorCode::_ENUM_MAX}.ascii()) != " MAX")) {
+        v3fatalSrc("Enum table in V3ErrorCode::EC_ascii() is munged");
+    }
+}
+
+string V3Error::lineStr(const char* filename, int lineno) VL_PURE {
+    std::ostringstream out;
+    const char* const fnslashp = std::strrchr(filename, '/');
+    if (fnslashp) filename = fnslashp + 1;
+    out << filename << ":" << std::dec << lineno << ":";
+    const char* const spaces = "                    ";
+    size_t numsp = out.str().length();
+    if (numsp > 20) numsp = 20;
+    out << (spaces + numsp);
+    return out.str();
+}
+
+void V3Error::abortIfWarnings() {
+    const bool exwarn = warnFatal() && warnCount();
+    if (errorCount() && exwarn) {
+        v3fatalExit("Exiting due to " << std::dec << V3Error::s().errorCount() << " error(s), "  //
+                                      << V3Error::s().warnCount() << " warning(s)\n");
+    } else if (errorCount()) {
+        v3fatalExit("Exiting due to " << std::dec << V3Error::s().errorCount() << " error(s)\n");
+    } else if (exwarn) {
+        v3fatalExit("Exiting due to " << std::dec << V3Error::s().warnCount() << " warning(s)\n");
+    }
+}
+
+//======================================================================
+// Abort/exit
+
+void V3Error::vlAbort() {
+    VL_GCOV_DUMP();
+    std::abort();
+}
diff --git a/src/V3Error.h b/src/V3Error.h
index 1434dbf03..c5aecadee 100644
--- a/src/V3Error.h
+++ b/src/V3Error.h
@@ -20,8 +20,13 @@
 #include "config_build.h"
 #include "verilatedos.h"
 
+#include "verilated_threads.h"
+
 // Limited V3 headers here - this is a base class for Vlc etc
 #include "V3String.h"
+#ifndef V3ERROR_NO_GLOBAL_
+#include "V3ThreadPool.h"
+#endif
 
 #include <array>
 #include <bitset>
@@ -268,78 +273,224 @@ inline std::ostream& operator<<(std::ostream& os, const V3ErrorCode& rhs) {
 }
 
 // ######################################################################
-
-class V3Error final {
-    // Base class for any object that wants debugging and error reporting
-
+class V3ErrorGuarded final {
+    // Should only be used by V3ErrorGuarded::m_mutex is already locked
+    // contains guarded members
+public:
     using MessagesSet = std::set<std::string>;
     using ErrorExitCb = void (*)(void);
 
 private:
-    static bool s_describedWarnings;  // Told user how to disable warns
-    static bool s_describedWeb;  // Told user to see web
-    static std::array<bool, V3ErrorCode::_ENUM_MAX>
-        s_describedEachWarn;  // Told user specifics about this warning
-    static std::array<bool, V3ErrorCode::_ENUM_MAX>
-        s_pretendError;  // Pretend this warning is an error
-    static int s_debugDefault;  // Option: --debugi Default debugging level
-    static int s_errorLimit;  // Option: --error-limit Number of errors before exit
-    static bool s_warnFatal;  // Option: --warnFatal Warnings are fatal
-    static int s_errCount;  // Error count
-    static int s_warnCount;  // Warning count
-    static int s_tellManual;  // Tell user to see manual, 0=not yet, 1=doit, 2=disable
-    static std::ostringstream s_errorStr;  // Error string being formed
-    static V3ErrorCode s_errorCode;  // Error string being formed will abort
-    static bool s_errorContexted;  // Error being formed got context
-    static bool s_errorSuppressed;  // Error being formed should be suppressed
-    static MessagesSet s_messages;  // What errors we've outputted
-    static ErrorExitCb s_errorExitCb;  // Callback when error occurs for dumping
-
     static constexpr unsigned MAX_ERRORS = 50;  // Fatal after this may errors
 
+    bool m_describedWarnings VL_GUARDED_BY(m_mutex) = false;  // Told user how to disable warns
+    // Tell user to see manual, 0=not yet, 1=doit, 2=disable
+    int m_tellManual VL_GUARDED_BY(m_mutex) = 0;
+    V3ErrorCode m_errorCode VL_GUARDED_BY(m_mutex)
+        = V3ErrorCode::EC_FATAL;  // Error string being formed will abort
+    bool m_errorSuppressed VL_GUARDED_BY(m_mutex)
+        = false;  // Error being formed should be suppressed
+    MessagesSet m_messages VL_GUARDED_BY(m_mutex);  // What errors we've outputted
+    ErrorExitCb m_errorExitCb VL_GUARDED_BY(m_mutex)
+        = nullptr;  // Callback when error occurs for dumping
+    bool m_errorContexted VL_GUARDED_BY(m_mutex) = false;  // Error being formed got context
+    int m_warnCount VL_GUARDED_BY(m_mutex) = 0;  // Warning count
+    int m_errCount VL_GUARDED_BY(m_mutex) = 0;  // Error count
+    // Pretend this warning is an error
+    std::array<bool, V3ErrorCode::_ENUM_MAX> m_pretendError VL_GUARDED_BY(m_mutex);
+    bool m_describedWeb VL_GUARDED_BY(m_mutex) = false;  // Told user to see web
+    // Told user specifics about this warning
+    std::array<bool, V3ErrorCode::_ENUM_MAX> m_describedEachWarn VL_GUARDED_BY(m_mutex);
+    int m_debugDefault = 0;  // Option: --debugi Default debugging level
+    int m_errorLimit VL_GUARDED_BY(m_mutex)
+        = MAX_ERRORS;  // Option: --error-limit Number of errors before exit
+    bool m_warnFatal VL_GUARDED_BY(m_mutex) = true;  // Option: --warnFatal Warnings are fatal
+    std::ostringstream m_errorStr VL_GUARDED_BY(m_mutex);  // Error string being formed
+public:
+    VerilatedMutex m_mutex;  // Make sure only single thread is in class
+
+    string msgPrefix() VL_REQUIRES(m_mutex);  // returns %Error/%Warn
+    string warnMore() VL_REQUIRES(m_mutex);
+    void execErrorExitCb() VL_REQUIRES(m_mutex) {
+        if (m_errorExitCb) m_errorExitCb();
+    }
+    void errorExitCb(ErrorExitCb cb) VL_REQUIRES(m_mutex) { m_errorExitCb = cb; }
+    ErrorExitCb errorExitCb() VL_REQUIRES(m_mutex) { return m_errorExitCb; }
+    bool isError(V3ErrorCode code, bool supp) VL_REQUIRES(m_mutex);
+    void vlAbortOrExit() VL_REQUIRES(m_mutex);
+    void errorContexted(bool flag) VL_REQUIRES(m_mutex) { m_errorContexted = flag; }
+    void incWarnings() VL_REQUIRES(m_mutex) { m_warnCount++; }
+    void incErrors() VL_REQUIRES(m_mutex) {
+        m_errCount++;
+        if (errorCount() == errorLimit()) {  // Not >= as would otherwise recurse
+            v3errorEnd(
+                (v3errorPrep(V3ErrorCode::EC_FATALEXIT),
+                 (v3errorStr() << "Exiting due to too many errors encountered; --error-limit="
+                               << errorCount() << endl),
+                 v3errorStr()));
+            assert(0);  // LCOV_EXCL_LINE
+            VL_UNREACHABLE;
+        }
+    }
+    int errorCount() VL_REQUIRES(m_mutex) { return m_errCount; }
+    bool pretendError(int errorCode) VL_REQUIRES(m_mutex) { return m_pretendError[errorCode]; }
+    void pretendError(V3ErrorCode code, bool flag) VL_REQUIRES(m_mutex) {
+        if (code == V3ErrorCode::WIDTH) {
+            m_pretendError[V3ErrorCode::WIDTHTRUNC] = flag;
+            m_pretendError[V3ErrorCode::WIDTHEXPAND] = flag;
+            m_pretendError[V3ErrorCode::WIDTHXZEXPAND] = flag;
+        }
+        m_pretendError[code] = flag;
+    }
+    void debugDefault(int level) VL_MT_UNSAFE { m_debugDefault = level; }
+    int debugDefault() VL_MT_SAFE { return m_debugDefault; }
+    void errorLimit(int level) VL_REQUIRES(m_mutex) { m_errorLimit = level; }
+    int errorLimit() VL_REQUIRES(m_mutex) { return m_errorLimit; }
+    void warnFatal(bool flag) VL_REQUIRES(m_mutex) { m_warnFatal = flag; }
+    bool warnFatal() VL_REQUIRES(m_mutex) { return m_warnFatal; }
+    void v3errorPrep(V3ErrorCode code) VL_REQUIRES(m_mutex) {
+        m_errorStr.str("");
+        m_errorCode = code;
+        m_errorContexted = false;
+        m_errorSuppressed = false;
+    }
+    std::ostringstream& v3errorStr() VL_REQUIRES(m_mutex) { return m_errorStr; }
+    V3ErrorCode errorCode() VL_REQUIRES(m_mutex) { return m_errorCode; }
+    bool errorContexted() VL_REQUIRES(m_mutex) { return m_errorContexted; }
+    int warnCount() VL_REQUIRES(m_mutex) { return m_warnCount; }
+    bool errorSuppressed() VL_REQUIRES(m_mutex) { return m_errorSuppressed; }
+    void errorSuppressed(bool flag) VL_REQUIRES(m_mutex) { m_errorSuppressed = flag; }
+    bool describedWeb() VL_REQUIRES(m_mutex) { return m_describedWeb; }
+    void describedWeb(bool flag) VL_REQUIRES(m_mutex) { m_describedWeb = flag; }
+    bool describedEachWarn(V3ErrorCode code) VL_REQUIRES(m_mutex) {
+        return m_describedEachWarn[code];
+    }
+    void describedEachWarn(V3ErrorCode code, bool flag) VL_REQUIRES(m_mutex) {
+        m_describedEachWarn[code] = flag;
+    }
+    bool describedWarnings() VL_REQUIRES(m_mutex) { return m_describedWarnings; }
+    void describedWarnings(bool flag) VL_REQUIRES(m_mutex) { m_describedWarnings = flag; }
+    int tellManual() VL_REQUIRES(m_mutex) { return m_tellManual; }
+    void tellManual(int level) VL_REQUIRES(m_mutex) { m_tellManual = level; }
+    void v3errorEnd(std::ostringstream& sstr, const string& extra = "") VL_REQUIRES(m_mutex);
+    void suppressThisWarning() VL_REQUIRES(m_mutex);
+    string warnContextNone() VL_REQUIRES(m_mutex) {
+        errorContexted(true);
+        return "";
+    }
+};
+
+// ######################################################################
+class V3Error final {
+    // Base class for any object that wants debugging and error reporting
+private:
+    // CONSTRUCTORS
     V3Error() {
         std::cerr << ("Static class");
         V3Error::vlAbort();
     }
 
 public:
-    // CONSTRUCTORS
+    static V3ErrorGuarded& s() VL_MT_SAFE {  // Singleton
+        static V3ErrorGuarded s_s;
+        return s_s;
+    }
+
     // ACCESSORS
-    static void debugDefault(int level) { s_debugDefault = level; }
-    static int debugDefault() VL_MT_SAFE { return s_debugDefault; }
-    static void errorLimit(int level) { s_errorLimit = level; }
-    static int errorLimit() VL_MT_SAFE { return s_errorLimit; }
-    static void warnFatal(bool flag) { s_warnFatal = flag; }
-    static bool warnFatal() { return s_warnFatal; }
-    static string msgPrefix();  // returns %Error/%Warn
-    static int errorCount() VL_MT_SAFE { return s_errCount; }
-    static int warnCount() { return s_warnCount; }
-    static bool errorContexted() VL_MT_SAFE { return s_errorContexted; }
-    static void errorContexted(bool flag) { s_errorContexted = flag; }
+    static void debugDefault(int level) VL_MT_UNSAFE { s().debugDefault(level); }
+    static int debugDefault() VL_MT_SAFE { return s().debugDefault(); }
+    static void errorLimit(int level) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().errorLimit(level);
+    }
+    static int errorLimit() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().errorLimit();
+    }
+    static void warnFatal(bool flag) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().warnFatal(flag);
+    }
+    static bool warnFatal() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().warnFatal();
+    }
+    // returns %Error/%Warn
+    static string msgPrefix() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().msgPrefix();
+    }
+    static int errorCount() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().errorCount();
+    }
+    static bool pretendError(int errorCode) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().pretendError(errorCode);
+    }
+    static int warnCount() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().warnCount();
+    }
+    static bool errorContexted() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().errorContexted();
+    }
+    static void errorContexted(bool flag) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().errorContexted(flag);
+    }
+    static void describedEachWarn(V3ErrorCode code, bool flag) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().describedEachWarn(code, flag);
+    }
     // METHODS
-    static void incErrors();
-    static void incWarnings() { s_warnCount++; }
+    static void incErrors() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().incErrors();
+    }
+    static void incWarnings() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().incWarnings();
+    }
     static void init();
     static void abortIfErrors() {
         if (errorCount()) abortIfWarnings();
     }
     static void abortIfWarnings();
-    static void suppressThisWarning();  // Suppress next %Warn if user has it off
-    static void pretendError(V3ErrorCode code, bool flag) {
-        if (code == V3ErrorCode::WIDTH) {
-            s_pretendError[V3ErrorCode::WIDTHTRUNC] = flag;
-            s_pretendError[V3ErrorCode::WIDTHEXPAND] = flag;
-            s_pretendError[V3ErrorCode::WIDTHXZEXPAND] = flag;
-        }
-        s_pretendError[code] = flag;
+    // Suppress next %Warn if user has it off
+    static void suppressThisWarning() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().suppressThisWarning();
+    }
+    static void pretendError(V3ErrorCode code, bool flag) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().pretendError(code, flag);
+    }
+    static string lineStr(const char* filename, int lineno) VL_PURE;
+    static V3ErrorCode errorCode() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().errorCode();
+    }
+    static void errorExitCb(V3ErrorGuarded::ErrorExitCb cb) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().errorExitCb(cb);
     }
-    static bool isError(V3ErrorCode code, bool supp);
-    static string lineStr(const char* filename, int lineno);
-    static V3ErrorCode errorCode() VL_MT_SAFE { return s_errorCode; }
-    static void errorExitCb(ErrorExitCb cb) { s_errorExitCb = cb; }
 
     // When printing an error/warning, print prefix for multiline message
-    static string warnMore();
+    static string warnMore() VL_REQUIRES(s().m_mutex) { return s().warnMore(); }
+    // This function should only be used when it is impossible to
+    // generate whole error message inside v3warn macros and it needs to be
+    // streamed directly to cerr.
+    // Use with caution as this function isn't MT_SAFE.
+    static string warnMoreStandalone() VL_EXCLUDES(s().m_mutex) VL_MT_UNSAFE {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().warnMore();
+    }
+    // This function marks place in error message from which point message
+    // should be printed after information on the error code.
+    // The post-processing is done in v3errorEnd function.
+    static string warnAdditionalInfo() VL_MT_SAFE { return "__WARNADDITIONALINFO__"; }
     /// When building an error, don't show context info
     static string warnContextNone() {
         V3Error::errorContexted(true);
@@ -348,37 +499,66 @@ public:
 
     // Internals for v3error()/v3fatal() macros only
     // Error end takes the string stream to output, be careful to seek() as needed
-    static void v3errorPrep(V3ErrorCode code) {
-        s_errorStr.str("");
-        s_errorCode = code;
-        s_errorContexted = false;
-        s_errorSuppressed = false;
+    static void v3errorPrep(V3ErrorCode code) VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().v3errorPrep(code);
+    }
+    static std::ostringstream& v3errorStr() VL_MT_SAFE_EXCLUDES(s().m_mutex) {
+        const VerilatedLockGuard guard{s().m_mutex};
+        return s().v3errorStr();
     }
-    static std::ostringstream& v3errorStr() { return s_errorStr; }
-    static void vlAbortOrExit();
     static void vlAbort();
     // static, but often overridden in classes.
-    static void v3errorEnd(std::ostringstream& sstr, const string& extra = "");
+    static void v3errorEnd(std::ostringstream& sstr, const string& extra = "")
+        VL_MT_SAFE_EXCLUDES(s().m_mutex) VL_MT_SAFE {
+        const VerilatedLockGuard guard{s().m_mutex};
+        s().v3errorEnd(sstr, extra);
+    }
+    // We can't call 's().v3errorEnd' directly in 'v3ErrorEnd'/'v3errorEndFatal',
+    // due to bug in GCC (tested on 11.3.0 version with --enable-m32)
+    // causing internal error when backtrace is printed.
+    // Instead use this wrapper.
+    static void v3errorEndGuardedCall(std::ostringstream& sstr, const string& extra = "")
+        VL_REQUIRES(s().m_mutex) VL_MT_SAFE {
+        s().v3errorEnd(sstr, extra);
+    }
 };
 
 // Global versions, so that if the class doesn't define a operator, we get the functions anyways.
-inline void v3errorEnd(std::ostringstream& sstr) { V3Error::v3errorEnd(sstr); }
-inline void v3errorEndFatal(std::ostringstream& sstr) {
-    V3Error::v3errorEnd(sstr);
+inline void v3errorEnd(std::ostringstream& sstr) VL_REQUIRES(V3Error::s().m_mutex) VL_MT_SAFE {
+    V3Error::v3errorEndGuardedCall(sstr);
+}
+inline void v3errorEndFatal(std::ostringstream& sstr)
+    VL_REQUIRES(V3Error::s().m_mutex) VL_MT_SAFE {
+    V3Error::v3errorEndGuardedCall(sstr);
     assert(0);  // LCOV_EXCL_LINE
     VL_UNREACHABLE;
 }
 
+#ifndef V3ERROR_NO_GLOBAL_
+#define V3ErrorLockAndCheckStopRequested \
+    V3Error::s().m_mutex.lockCheckStopRequest( \
+        []() -> void { V3ThreadPool::s().waitIfStopRequested(); })
+#else
+#define V3ErrorLockAndCheckStopRequested V3Error::s().m_mutex.lock()
+#endif
+
 // Theses allow errors using << operators: v3error("foo"<<"bar");
 // Careful, you can't put () around msg, as you would in most macro definitions
 // Note the commas are the comma operator, not separating arguments. These are needed to ensure
 // evaluation order as otherwise we couldn't ensure v3errorPrep is called first.
+// Note: due to limitations of clang thread-safety analysis, we can't use
+// lock guard here, instead we are locking the mutex as first operation in temporary,
+// but we are unlocking the mutex after function using comma operator.
+// This way macros should also work when they are in 'if' stmt without '{}'.
 #define v3warnCode(code, msg) \
-    v3errorEnd((V3Error::v3errorPrep(code), (V3Error::v3errorStr() << msg), V3Error::v3errorStr()))
-
+    v3errorEnd((V3ErrorLockAndCheckStopRequested, V3Error::s().v3errorPrep(code), \
+                (V3Error::s().v3errorStr() << msg), V3Error::s().v3errorStr())), \
+        V3Error::s().m_mutex.unlock()
 #define v3warnCodeFatal(code, msg) \
-    v3errorEndFatal( \
-        (V3Error::v3errorPrep(code), (V3Error::v3errorStr() << msg), V3Error::v3errorStr()))
+    v3errorEndFatal((V3ErrorLockAndCheckStopRequested, V3Error::s().v3errorPrep(code), \
+                     (V3Error::s().v3errorStr() << msg), V3Error::s().v3errorStr())), \
+        V3Error::s().m_mutex.unlock()
 #define v3warn(code, msg) v3warnCode(V3ErrorCode::code, msg)
 #define v3info(msg) v3warnCode(V3ErrorCode::EC_INFO, msg)
 #define v3error(msg) v3warnCode(V3ErrorCode::EC_ERROR, msg)
@@ -391,8 +571,10 @@ inline void v3errorEndFatal(std::ostringstream& sstr) {
                     __FILE__ << ":" << std::dec << __LINE__ << ": " << msg)
 // Use this when normal v3fatal is called in static method that overrides fileline.
 #define v3fatalStatic(msg) \
-    (::v3errorEndFatal((V3Error::v3errorPrep(V3ErrorCode::EC_FATAL), \
-                        (V3Error::v3errorStr() << msg), V3Error::v3errorStr())))
+    (::v3errorEndFatal((V3ErrorLockAndCheckStopRequested, \
+                        V3Error::s().v3errorPrep(V3ErrorCode::EC_FATAL), \
+                        (V3Error::s().v3errorStr() << msg), V3Error::s().v3errorStr()))), \
+        V3Error::s().m_mutex.unlock()
 
 #define UINFO(level, stmsg) \
     do { \
diff --git a/src/V3FileLine.cpp b/src/V3FileLine.cpp
index 026bd2823..9bee6a86a 100644
--- a/src/V3FileLine.cpp
+++ b/src/V3FileLine.cpp
@@ -322,7 +322,7 @@ string FileLine::asciiLineCol() const {
             + "-" + cvtToStr(lastColumn()) + "[" + (m_contentp ? m_contentp->ascii() : "ct0") + "+"
             + cvtToStr(m_contentLineno) + "]");
 }
-string FileLine::ascii() const VL_MT_SAFE {
+string FileLine::ascii() const {
     // For most errors especially in the parser the lastLineno is more accurate than firstLineno
     return filename() + ":" + cvtToStr(lastLineno()) + ":" + cvtToStr(firstColumn());
 }
@@ -369,7 +369,7 @@ void FileLine::warnUnusedOff(bool flag) {
     warnOff(V3ErrorCode::UNUSEDSIGNAL, flag);
 }
 
-bool FileLine::warnIsOff(V3ErrorCode code) const VL_MT_SAFE {
+bool FileLine::warnIsOff(V3ErrorCode code) const {
     if (!msgEn().test(code)) return true;
     if (!defaultFileLine().msgEn().test(code)) return true;  // Global overrides local
     if ((code.lintError() || code.styleError()) && !msgEn().test(V3ErrorCode::I_LINT)) {
@@ -380,7 +380,8 @@ bool FileLine::warnIsOff(V3ErrorCode code) const VL_MT_SAFE {
 }
 
 // cppverilator-suppress constParameter
-void FileLine::v3errorEnd(std::ostringstream& sstr, const string& extra) {
+void FileLine::v3errorEnd(std::ostringstream& sstr, const string& extra)
+    VL_REQUIRES(V3Error::s().m_mutex) {
     std::ostringstream nsstr;
     if (lastLineno()) nsstr << this;
     nsstr << sstr.str();
@@ -390,29 +391,33 @@ void FileLine::v3errorEnd(std::ostringstream& sstr, const string& extra) {
         lstr << std::setw(ascii().length()) << " "
              << ": " << extra;
     }
-    m_waive = V3Config::waive(this, V3Error::errorCode(), sstr.str());
-    if (warnIsOff(V3Error::errorCode()) || m_waive) {
-        V3Error::suppressThisWarning();
-    } else if (!V3Error::errorContexted()) {
+    m_waive = V3Config::waive(this, V3Error::s().errorCode(), sstr.str());
+    if (warnIsOff(V3Error::s().errorCode()) || m_waive) {
+        V3Error::s().suppressThisWarning();
+    } else if (!V3Error::s().errorContexted()) {
         nsstr << warnContextPrimary();
     }
-    if (!m_waive) V3Waiver::addEntry(V3Error::errorCode(), filename(), sstr.str());
-    V3Error::v3errorEnd(nsstr, lstr.str());
+    if (!m_waive) V3Waiver::addEntry(V3Error::s().errorCode(), filename(), sstr.str());
+    V3Error::s().v3errorEnd(nsstr, lstr.str());
 }
 
-string FileLine::warnMore() const {
+string FileLine::warnMore() const VL_REQUIRES(V3Error::s().m_mutex) {
     if (lastLineno()) {
-        return V3Error::warnMore() + string(ascii().size(), ' ') + ": ";
+        return V3Error::s().warnMore() + string(ascii().size(), ' ') + ": ";
     } else {
-        return V3Error::warnMore();
+        return V3Error::s().warnMore();
     }
 }
-string FileLine::warnOther() const VL_MT_SAFE {
+string FileLine::warnOther() const VL_REQUIRES(V3Error::s().m_mutex) {
     if (lastLineno()) {
-        return V3Error::warnMore() + ascii() + ": ";
+        return V3Error::s().warnMore() + ascii() + ": ";
     } else {
-        return V3Error::warnMore();
+        return V3Error::s().warnMore();
     }
+};
+string FileLine::warnOtherStandalone() const VL_EXCLUDES(V3Error::s().m_mutex) VL_MT_UNSAFE {
+    const VerilatedLockGuard guard{V3Error::s().m_mutex};
+    return warnOther();
 }
 
 string FileLine::source() const VL_MT_SAFE {
@@ -435,8 +440,7 @@ string FileLine::prettySource() const VL_MT_SAFE {
     return VString::spaceUnprintable(out);
 }
 
-string FileLine::warnContext(bool secondary) const VL_MT_SAFE {
-    V3Error::errorContexted(true);
+string FileLine::warnContext() const {
     if (!v3Global.opt.context()) return "";
     string out;
     if (firstLineno() == lastLineno() && firstColumn()) {
@@ -457,16 +461,18 @@ string FileLine::warnContext(bool secondary) const VL_MT_SAFE {
             out += "\n";
         }
     }
-    if (!secondary) {  // Avoid printing long paths on informational part of error
-        for (FileLine* parentFl = parent(); parentFl; parentFl = parentFl->parent()) {
-            if (parentFl->filenameIsGlobal()) break;
-            out += parentFl->warnOther() + "... note: In file included from "
-                   + parentFl->filebasename() + "\n";
-        }
-    }
     return out;
 }
 
+string FileLine::warnContextParent() const VL_REQUIRES(V3Error::s().m_mutex) {
+    string out;
+    for (FileLine* parentFl = parent(); parentFl; parentFl = parentFl->parent()) {
+        if (parentFl->filenameIsGlobal()) break;
+        out += parentFl->warnOther() + "... note: In file included from "
+               + parentFl->filebasename() + "\n";
+    }
+    return out;
+}
 #ifdef VL_LEAK_CHECKS
 std::unordered_set<FileLine*> fileLineLeakChecks;
 
diff --git a/src/V3FileLine.h b/src/V3FileLine.h
index e462e2065..42c72f25b 100644
--- a/src/V3FileLine.h
+++ b/src/V3FileLine.h
@@ -314,24 +314,30 @@ public:
     void modifyWarnOff(V3ErrorCode code, bool flag) { warnOff(code, flag); }
 
     // OPERATORS
-    void v3errorEnd(std::ostringstream& str, const string& extra = "");
-    void v3errorEndFatal(std::ostringstream& str) VL_ATTR_NORETURN {
+    void v3errorEnd(std::ostringstream& str, const string& extra = "")
+        VL_REQUIRES(V3Error::s().m_mutex);
+    void v3errorEndFatal(std::ostringstream& str) VL_ATTR_NORETURN
+        VL_REQUIRES(V3Error::s().m_mutex) {
         v3errorEnd(str);
         assert(0);  // LCOV_EXCL_LINE
         VL_UNREACHABLE;
     }
     /// When building an error, prefix for printing continuation lines
     /// e.g. information referring to the same FileLine as before
-    string warnMore() const;
+    string warnMore() const VL_REQUIRES(V3Error::s().m_mutex);
     /// When building an error, prefix for printing secondary information
     /// from a different FileLine than the original error
-    string warnOther() const VL_MT_SAFE;
+    string warnOther() const VL_REQUIRES(V3Error::s().m_mutex);
+    string warnOtherStandalone() const VL_EXCLUDES(V3Error::s().m_mutex) VL_MT_UNSAFE;
     /// When building an error, current location in include etc
     /// If not used in a given error, automatically pasted at end of error
-    string warnContextPrimary() const VL_MT_SAFE { return warnContext(false); }
+    string warnContextPrimary() const VL_REQUIRES(V3Error::s().m_mutex) {
+        V3Error::s().errorContexted(true);
+        return warnContext() + warnContextParent();
+    }
     /// When building an error, additional location for additional references
     /// Simplified information vs warnContextPrimary() to make dump clearer
-    string warnContextSecondary() const VL_MT_SAFE { return warnContext(true); }
+    string warnContextSecondary() const { return warnContext(); }
     bool operator==(const FileLine& rhs) const {
         return (m_firstLineno == rhs.m_firstLineno && m_firstColumn == rhs.m_firstColumn
                 && m_lastLineno == rhs.m_lastLineno && m_lastColumn == rhs.m_lastColumn
@@ -354,7 +360,8 @@ public:
     }
 
 private:
-    string warnContext(bool secondary) const VL_MT_SAFE;
+    string warnContext() const;
+    string warnContextParent() const VL_REQUIRES(V3Error::s().m_mutex);
     const MsgEnBitSet& msgEn() const VL_MT_SAFE { return singleton().msgEn(m_msgEnIdx); }
 };
 std::ostream& operator<<(std::ostream& os, FileLine* fileline);
diff --git a/src/V3Graph.cpp b/src/V3Graph.cpp
index 202cd4587..27b37adf2 100644
--- a/src/V3Graph.cpp
+++ b/src/V3Graph.cpp
@@ -129,7 +129,7 @@ V3GraphEdge* V3GraphVertex::findConnectingEdgep(GraphWay way, const V3GraphVerte
 }
 
 // cppcheck-has-bug-suppress constParameter
-void V3GraphVertex::v3errorEnd(std::ostringstream& str) const {
+void V3GraphVertex::v3errorEnd(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex) {
     std::ostringstream nsstr;
     nsstr << str.str();
     if (debug()) {
@@ -139,10 +139,11 @@ void V3GraphVertex::v3errorEnd(std::ostringstream& str) const {
     if (FileLine* const flp = fileline()) {
         flp->v3errorEnd(nsstr);
     } else {
-        V3Error::v3errorEnd(nsstr);
+        V3Error::s().v3errorEnd(nsstr);
     }
 }
-void V3GraphVertex::v3errorEndFatal(std::ostringstream& str) const {
+void V3GraphVertex::v3errorEndFatal(std::ostringstream& str) const
+    VL_REQUIRES(V3Error::s().m_mutex) {
     v3errorEnd(str);
     assert(0);  // LCOV_EXCL_LINE
     VL_UNREACHABLE;
diff --git a/src/V3Graph.h b/src/V3Graph.h
index 132221767..27aa13908 100644
--- a/src/V3Graph.h
+++ b/src/V3Graph.h
@@ -248,8 +248,8 @@ public:
     V3GraphEdge* beginp(GraphWay way) const { return way.forward() ? outBeginp() : inBeginp(); }
     // METHODS
     /// Error reporting
-    void v3errorEnd(std::ostringstream& str) const;
-    void v3errorEndFatal(std::ostringstream& str) const;
+    void v3errorEnd(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex);
+    void v3errorEndFatal(std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex);
     /// Edges are routed around this vertex to point from "from" directly to "to"
     void rerouteEdges(V3Graph* graphp);
     /// Find the edge connecting ap and bp, where bp is wayward from ap.
diff --git a/src/V3LinkLevel.cpp b/src/V3LinkLevel.cpp
index cfa121649..9718b9b67 100644
--- a/src/V3LinkLevel.cpp
+++ b/src/V3LinkLevel.cpp
@@ -59,15 +59,23 @@ void V3LinkLevel::modSortByLevel() {
     if (tops.size() >= 2) {
         const AstNode* const secp = tops[1];  // Complain about second one, as first often intended
         if (!secp->fileline()->warnIsOff(V3ErrorCode::MULTITOP)) {
+            auto warnTopModules = [](std::string warnMore, ModVec tops)
+                                      VL_REQUIRES(V3Error::s().m_mutex) -> std::string {
+                std::stringstream ss;
+                for (AstNode* alsop : tops) {
+                    ss << warnMore << "... Top module " << alsop->prettyNameQ() << endl
+                       << alsop->warnContextSecondary();
+                }
+                return ss.str();
+            };
+
             secp->v3warn(MULTITOP, "Multiple top level modules\n"
                                        << secp->warnMore()
                                        << "... Suggest see manual; fix the duplicates, or use "
                                           "--top-module to select top."
-                                       << V3Error::warnContextNone());
-            for (AstNode* alsop : tops) {
-                std::cerr << secp->warnMore() << "... Top module " << alsop->prettyNameQ() << endl
-                          << alsop->warnContextSecondary();
-            }
+                                       << V3Error::s().warnContextNone()
+                                       << V3Error::warnAdditionalInfo()
+                                       << warnTopModules(secp->warnMore(), tops));
         }
     }
 
diff --git a/src/V3Number.cpp b/src/V3Number.cpp
index 4772120f9..758805db1 100644
--- a/src/V3Number.cpp
+++ b/src/V3Number.cpp
@@ -76,7 +76,7 @@ constexpr int MAX_SPRINTF_DOUBLE_SIZE
 //======================================================================
 // Errors
 
-void V3Number::v3errorEnd(const std::ostringstream& str) const VL_MT_SAFE {
+void V3Number::v3errorEnd(const std::ostringstream& str) const VL_REQUIRES(V3Error::s().m_mutex) {
     std::ostringstream nsstr;
     nsstr << str.str();
     if (m_nodep) {
@@ -84,11 +84,12 @@ void V3Number::v3errorEnd(const std::ostringstream& str) const VL_MT_SAFE {
     } else if (m_fileline) {
         m_fileline->v3errorEnd(nsstr);
     } else {
-        V3Error::v3errorEnd(nsstr);
+        V3Error::s().v3errorEnd(nsstr);
     }
 }
 
-void V3Number::v3errorEndFatal(const std::ostringstream& str) const VL_MT_SAFE {
+void V3Number::v3errorEndFatal(const std::ostringstream& str) const
+    VL_REQUIRES(V3Error::s().m_mutex) {
     v3errorEnd(str);
     assert(0);  // LCOV_EXCL_LINE
     VL_UNREACHABLE;
diff --git a/src/V3Number.h b/src/V3Number.h
index b58806705..f9b6bdc05 100644
--- a/src/V3Number.h
+++ b/src/V3Number.h
@@ -566,8 +566,9 @@ private:
     }
 
 public:
-    void v3errorEnd(const std::ostringstream& sstr) const;
-    void v3errorEndFatal(const std::ostringstream& sstr) const VL_ATTR_NORETURN;
+    void v3errorEnd(const std::ostringstream& sstr) const VL_REQUIRES(V3Error::s().m_mutex);
+    void v3errorEndFatal(const std::ostringstream& sstr) const VL_ATTR_NORETURN
+        VL_REQUIRES(V3Error::s().m_mutex);
     void width(int width, bool sized = true) {
         m_data.m_sized = sized;
         m_data.resize(width);
diff --git a/src/V3Options.cpp b/src/V3Options.cpp
index 24f16d6d5..71097d95b 100644
--- a/src/V3Options.cpp
+++ b/src/V3Options.cpp
@@ -572,26 +572,28 @@ string V3Options::filePath(FileLine* fl, const string& modname, const string& la
 void V3Options::filePathLookedMsg(FileLine* fl, const string& modname) {
     static bool shown_notfound_msg = false;
     if (modname.find("__Vhsh") != string::npos) {
-        std::cerr << V3Error::warnMore() << "... Unsupported: Name is longer than 127 characters;"
+        std::cerr << V3Error::warnMoreStandalone()
+                  << "... Unsupported: Name is longer than 127 characters;"
                   << " automatic file lookup not supported.\n";
-        std::cerr << V3Error::warnMore() << "... Suggest putting filename with this module/package"
+        std::cerr << V3Error::warnMoreStandalone()
+                  << "... Suggest putting filename with this module/package"
                   << " onto command line instead.\n";
     } else if (!shown_notfound_msg) {
         shown_notfound_msg = true;
         if (m_impp->m_incDirUsers.empty()) {
             fl->v3error("This may be because there's no search path specified with -I<dir>.");
         }
-        std::cerr << V3Error::warnMore() << "... Looked in:" << endl;
+        std::cerr << V3Error::warnMoreStandalone() << "... Looked in:" << endl;
         for (const string& dir : m_impp->m_incDirUsers) {
             for (const string& ext : m_impp->m_libExtVs) {
                 const string fn = V3Os::filenameFromDirBase(dir, modname + ext);
-                std::cerr << V3Error::warnMore() << "     " << fn << endl;
+                std::cerr << V3Error::warnMoreStandalone() << "     " << fn << endl;
             }
         }
         for (const string& dir : m_impp->m_incDirFallbacks) {
             for (const string& ext : m_impp->m_libExtVs) {
                 const string fn = V3Os::filenameFromDirBase(dir, modname + ext);
-                std::cerr << V3Error::warnMore() << "     " << fn << endl;
+                std::cerr << V3Error::warnMoreStandalone() << "     " << fn << endl;
             }
         }
     }
diff --git a/src/V3PreProc.cpp b/src/V3PreProc.cpp
index 6a45c3e22..ef662dc17 100644
--- a/src/V3PreProc.cpp
+++ b/src/V3PreProc.cpp
@@ -185,7 +185,9 @@ public:
     // For getline()
     string m_lineChars;  ///< Characters left for next line
 
-    void v3errorEnd(std::ostringstream& str) { fileline()->v3errorEnd(str); }
+    void v3errorEnd(std::ostringstream& str) VL_REQUIRES(V3Error::s().m_mutex) {
+        fileline()->v3errorEnd(str);
+    }
 
     static const char* tokenName(int tok);
     void debugToken(int tok, const char* cmtp);
diff --git a/src/V3SchedAcyclic.cpp b/src/V3SchedAcyclic.cpp
index 4d196a068..5e085f59d 100644
--- a/src/V3SchedAcyclic.cpp
+++ b/src/V3SchedAcyclic.cpp
@@ -99,13 +99,13 @@ class Graph final : public V3Graph {
         // TODO: 'typeName' is an internal thing. This should be more human readable.
         if (LogicVertex* const lvtxp = dynamic_cast<LogicVertex*>(vtxp)) {
             AstNode* const logicp = lvtxp->logicp();
-            std::cerr << logicp->fileline()->warnOther()
+            std::cerr << logicp->fileline()->warnOtherStandalone()
                       << "     Example path: " << logicp->typeName() << endl;
         } else {
             VarVertex* const vvtxp = dynamic_cast<VarVertex*>(vtxp);
             UASSERT(vvtxp, "Cannot be anything else");
             AstVarScope* const vscp = vvtxp->vscp();
-            std::cerr << vscp->fileline()->warnOther()
+            std::cerr << vscp->fileline()->warnOtherStandalone()
                       << "     Example path: " << vscp->prettyName() << endl;
         }
     }
@@ -289,7 +289,7 @@ void reportLoopVars(Graph* graphp, VarVertex* vvtxp) {
             if (i == candidates.size()) break;
             const Candidate& candidate = candidates[i];
             AstVar* const varp = candidate.first->varp();
-            std::cerr << V3Error::warnMore() << "    " << varp->fileline() << " "
+            std::cerr << V3Error::warnMoreStandalone() << "    " << varp->fileline() << " "
                       << varp->prettyName() << ", width " << std::dec << varp->width()
                       << ", circular fanout " << candidate.second;
             if (V3SplitVar::canSplitVar(varp)) {
@@ -301,19 +301,19 @@ void reportLoopVars(Graph* graphp, VarVertex* vvtxp) {
     };
 
     // Widest variables
-    std::cerr << V3Error::warnMore() << "... Widest variables candidate to splitting:\n";
+    std::cerr << V3Error::warnMoreStandalone() << "... Widest variables candidate to splitting:\n";
     reportFirst10([](const Candidate& a, const Candidate& b) {
         return a.first->varp()->width() > b.first->varp()->width();
     });
 
     // Highest fanout
-    std::cerr << V3Error::warnMore() << "... Candidates with the highest fanout:\n";
+    std::cerr << V3Error::warnMoreStandalone() << "... Candidates with the highest fanout:\n";
     reportFirst10([](const Candidate& a, const Candidate& b) {  //
         return a.second > b.second;
     });
 
     if (splittable) {
-        std::cerr << V3Error::warnMore()
+        std::cerr << V3Error::warnMoreStandalone()
                   << "... Suggest add /*verilator split_var*/ to appropriate variables above."
                   << std::endl;
     }
diff --git a/src/V3SymTable.h b/src/V3SymTable.h
index d81b0f276..9d723a15e 100644
--- a/src/V3SymTable.h
+++ b/src/V3SymTable.h
@@ -266,7 +266,7 @@ public:
             }
         }
         if (scopes == "") scopes = "<no instances found>";
-        std::cerr << V3Error::warnMore() << "... Known scopes under '" << prettyName
+        std::cerr << V3Error::warnMoreStandalone() << "... Known scopes under '" << prettyName
                   << "': " << scopes << endl;
         if (debug()) dumpSelf(std::cerr, "       KnownScope: ", 1);
     }
diff --git a/src/V3Width.cpp b/src/V3Width.cpp
index b672e8a80..55ef8763c 100644
--- a/src/V3Width.cpp
+++ b/src/V3Width.cpp
@@ -119,10 +119,12 @@ std::ostream& operator<<(std::ostream& str, const Castable& rhs) {
 }
 
 #define v3widthWarn(lhs, rhs, msg) \
-    v3errorEnd((V3Error::v3errorPrep((lhs) < (rhs)   ? V3ErrorCode::WIDTHTRUNC \
-                                     : (lhs) > (rhs) ? V3ErrorCode::WIDTHEXPAND \
-                                                     : V3ErrorCode::WIDTH), \
-                (V3Error::v3errorStr() << msg), V3Error::v3errorStr()))
+    v3errorEnd((V3Error::s().m_mutex.lock(), \
+                V3Error::s().v3errorPrep((lhs) < (rhs)   ? V3ErrorCode::WIDTHTRUNC \
+                                         : (lhs) > (rhs) ? V3ErrorCode::WIDTHEXPAND \
+                                                         : V3ErrorCode::WIDTH), \
+                (V3Error::s().v3errorStr() << msg), V3Error::s().v3errorStr())), \
+        V3Error::s().m_mutex.unlock()
 
 //######################################################################
 // Width state, as a visitor of each AstNode
diff --git a/test_regress/t/t_a5_attributes_src.pl b/test_regress/t/t_a5_attributes_src.pl
index 8c2447820..2c8fe4e9b 100755
--- a/test_regress/t/t_a5_attributes_src.pl
+++ b/test_regress/t/t_a5_attributes_src.pl
@@ -36,7 +36,7 @@ sub check {
             tee => 1,
             cmd => ["python3", "$root/nodist/clang_check_attributes --verilator-root=$root --cxxflags='$clang_args' $precompile_args $srcfiles_str"]);
 
-        file_grep($Self->{run_log_filename}, "Number of functions reported unsafe: 24");
+        file_grep($Self->{run_log_filename}, "Number of functions reported unsafe: 14");
     }
 
     run_clang_check();