diff --git a/include/verilatedos.h b/include/verilatedos.h index 17a49c263..2a0739ff5 100644 --- a/include/verilatedos.h +++ b/include/verilatedos.h @@ -58,6 +58,9 @@ # define VL_ATTR_PRINTF(fmtArgNum) __attribute__((format(printf, (fmtArgNum), (fmtArgNum) + 1))) # define VL_ATTR_PURE __attribute__((pure)) # define VL_ATTR_UNUSED __attribute__((unused)) +#ifndef VL_ATTR_WARN_UNUSED_RESULT +# define VL_ATTR_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) +#endif # if !defined(_WIN32) && !defined(__MINGW32__) // All VL_ATTR_WEAK symbols must be marked with the macOS -U linker flag in verilated.mk.in # define VL_ATTR_WEAK __attribute__((weak)) @@ -164,6 +167,9 @@ #ifndef VL_ATTR_UNUSED # define VL_ATTR_UNUSED ///< Attribute that function that may be never used #endif +#ifndef VL_ATTR_WARN_UNUSED_RESULT +# define VL_ATTR_WARN_UNUSED_RESULT ///< Attribute that return value of function must be used +#endif #ifndef VL_ATTR_WEAK # define VL_ATTR_WEAK ///< Attribute that function external that is optionally defined #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e1c9b8ae..eb0fcdcfc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,6 +62,7 @@ set(HEADERS V3DepthBlock.h V3Descope.h V3Dfg.h + V3DfgCache.h V3DfgOptimizer.h V3DfgPasses.h V3DfgPatternStats.h @@ -214,6 +215,7 @@ set(COMMON_SOURCES V3Descope.cpp V3Dfg.cpp V3DfgAstToDfg.cpp + V3DfgCache.cpp V3DfgDecomposition.cpp V3DfgDfgToAst.cpp V3DfgOptimizer.cpp diff --git a/src/Makefile_obj.in b/src/Makefile_obj.in index 50bf1f9da..37874cd4b 100644 --- a/src/Makefile_obj.in +++ b/src/Makefile_obj.in @@ -228,6 +228,7 @@ RAW_OBJS_PCH_ASTNOMT = \ V3Descope.o \ V3Dfg.o \ V3DfgAstToDfg.o \ + V3DfgCache.o \ V3DfgDecomposition.o \ V3DfgDfgToAst.o \ V3DfgOptimizer.o \ diff --git a/src/V3DfgCache.cpp b/src/V3DfgCache.cpp new file mode 100644 index 000000000..036578e2b --- /dev/null +++ b/src/V3DfgCache.cpp @@ -0,0 +1,51 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Dfg vertex cache to find existing vertices +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// Copyright 2003-2024 by Wilson Snyder. This program is free software; you +// can redistribute it and/or modify it under the terms of either the GNU +// Lesser General Public License Version 3 or the Perl Artistic License +// Version 2.0. +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// A cache for DfgGraph, to find existing vertices with identical inputs. +// +// Beware that if you use data-structure, you must invalidate the cache any +// time you change the inputs of an existing vertex, otherwise you will +// have a very bad day. +// +//************************************************************************* + +#include "V3DfgCache.h" + +namespace V3DfgCacheInternal { + +void V3DfgCache::cache(DfgVertex* vtxp) { + switch (vtxp->type()) { +#define VERTEX_CACHE_ADD_CASE(t) \ + case t::dfgType(): V3DfgCacheInternal::cache(m_cache##t, reinterpret_cast(vtxp)); break; + FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_ADD_CASE) +#undef VERTEX_CACHE_ADD_CASE + default: break; + } +} + +void V3DfgCache::invalidateByValue(DfgVertex* vtxp) { + switch (vtxp->type()) { +#define VERTEX_CACHE_ADD_CASE(t) \ + case t::dfgType(): \ + V3DfgCacheInternal::invalidateByValue(m_cache##t, reinterpret_cast(vtxp)); \ + break; + FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_ADD_CASE) +#undef VERTEX_CACHE_ADD_CASE + default: break; + } +} + +} // namespace V3DfgCacheInternal diff --git a/src/V3DfgCache.h b/src/V3DfgCache.h new file mode 100644 index 000000000..a85af80e0 --- /dev/null +++ b/src/V3DfgCache.h @@ -0,0 +1,402 @@ +// -*- mode: C++; c-file-style: "cc-mode" -*- +//************************************************************************* +// DESCRIPTION: Verilator: Dfg vertex cache to find existing vertices +// +// Code available from: https://verilator.org +// +//************************************************************************* +// +// Copyright 2003-2024 by Wilson Snyder. This program is free software; you +// can redistribute it and/or modify it under the terms of either the GNU +// Lesser General Public License Version 3 or the Perl Artistic License +// Version 2.0. +// SPDX-License-Identifier: LGPL-3.0-only OR Artistic-2.0 +// +//************************************************************************* +// +// A cache for DfgGraph, to find existing vertices with identical inputs. +// +// Beware that if you use this data-structure, you must invalidate the +// cache any time you change the inputs of an existing vertex, otherwise +// you will have a very bad day. +// +//************************************************************************* + +#ifndef VERILATOR_V3DFGCACHE_H_ +#define VERILATOR_V3DFGCACHE_H_ + +#include "V3Dfg.h" + +#include + +// Template specializatoins must be defiend before they are used, +// so this is implemented in a namespace + +namespace V3DfgCacheInternal { + +// Hash constants by value, everything else by identity +inline V3Hash vertexHash(const DfgVertex* vtxp) { + if (const DfgConst* const constp = vtxp->cast()) return constp->num().toHash(); + return V3Hash{reinterpret_cast(vtxp)}; +} + +// Constants are equal by value, everything else is equal by identity +inline bool vertexEqual(const DfgVertex* ap, const DfgVertex* bp) { + if (ap == bp) return true; + if (ap->type() != bp->type()) return false; + if (const DfgConst* const aConstp = ap->cast()) { + const DfgConst* const bConstp = bp->as(); + return aConstp->num().isCaseEq(bConstp->num()); + } + return false; +} + +class KeySel final { + const DfgVertex* const m_fromp; + const uint32_t m_lsb; + const uint32_t m_width; + +public: + KeySel(DfgVertex* fromp, uint32_t lsb, uint32_t width) + : m_fromp{fromp} + , m_lsb{lsb} + , m_width{width} {} + + struct Hash final { + size_t operator()(const KeySel& key) const { + V3Hash hash{vertexHash(key.m_fromp)}; + hash += key.m_lsb; + hash += key.m_width; + return hash.value(); + } + }; + + struct Equal final { + bool operator()(const KeySel& a, const KeySel& b) const { + return a.m_lsb == b.m_lsb && a.m_width == b.m_width + && vertexEqual(a.m_fromp, b.m_fromp); + } + }; +}; + +class KeyUnary final { + const DfgVertex* const m_source0p; + +public: + KeyUnary(DfgVertex* source0p) + : m_source0p{source0p} {} + + struct Hash final { + size_t operator()(const KeyUnary& key) const { // + return vertexHash(key.m_source0p).value(); + } + }; + + struct Equal final { + bool operator()(const KeyUnary& a, const KeyUnary& b) const { + return vertexEqual(a.m_source0p, b.m_source0p); + } + }; +}; + +class KeyBinary final { + const DfgVertex* const m_source0p; + const DfgVertex* const m_source1p; + +public: + KeyBinary(DfgVertex* source0p, DfgVertex* source1p) + : m_source0p{source0p} + , m_source1p{source1p} {} + + struct Hash final { + size_t operator()(const KeyBinary& key) const { + V3Hash hash{vertexHash(key.m_source0p)}; + hash += vertexHash(key.m_source1p); + return hash.value(); + } + }; + + struct Equal final { + bool operator()(const KeyBinary& a, const KeyBinary& b) const { + return vertexEqual(a.m_source0p, b.m_source0p) + && vertexEqual(a.m_source1p, b.m_source1p); + } + }; +}; + +class KeyTernary final { + const DfgVertex* const m_source0p; + const DfgVertex* const m_source1p; + const DfgVertex* const m_source2p; + +public: + KeyTernary(DfgVertex* source0p, DfgVertex* source1p, DfgVertex* source2p) + : m_source0p{source0p} + , m_source1p{source1p} + , m_source2p{source2p} {} + + struct Hash final { + size_t operator()(const KeyTernary& key) const { + V3Hash hash{vertexHash(key.m_source0p)}; + hash += vertexHash(key.m_source1p); + hash += vertexHash(key.m_source2p); + return hash.value(); + } + }; + + struct Equal final { + bool operator()(const KeyTernary& a, const KeyTernary& b) const { + return vertexEqual(a.m_source0p, b.m_source0p) + && vertexEqual(a.m_source1p, b.m_source1p) + && vertexEqual(a.m_source2p, b.m_source2p); + } + }; +}; + +template +using Cache = std::unordered_map; + +using CacheSel = Cache; +using CacheUnary = Cache; +using CacheBinary = Cache; +using CacheTernary = Cache; + +// These return a reference to the mapped entry, inserting a nullptr if not yet exists +inline DfgSel*& getEntry(CacheSel& cache, AstNodeDType* dtypep, DfgVertex* src0p, uint32_t lsb) { + UASSERT_OBJ(VN_IS(dtypep, BasicDType), dtypep, "non-packed has no 'width()'"); + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p, lsb, dtypep->width()), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +inline DfgVertexUnary*& getEntry(CacheUnary& cache, AstNodeDType*, DfgVertex* src0p) { + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +inline DfgVertexBinary*& getEntry(CacheBinary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p) { + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p, src1p), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +inline DfgVertexTernary*& getEntry(CacheTernary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p, DfgVertex* src2p) { + return cache + .emplace(std::piecewise_construct, // + std::forward_as_tuple(src0p, src1p, src2p), // + std::forward_as_tuple(nullptr)) + .first->second; +} + +// These return a reference to the mapped entry, inserting a nullptr if not yet exists +inline CacheSel::iterator find(CacheSel& cache, AstNodeDType* dtypep, DfgVertex* src0p, + uint32_t lsb) { + UASSERT_OBJ(VN_IS(dtypep, BasicDType), dtypep, "non-packed has no 'width()'"); + const uint32_t width = dtypep->width(); + return cache.find({src0p, lsb, width}); +} + +inline CacheUnary::iterator find(CacheUnary& cache, AstNodeDType*, DfgVertex* src0p) { + return cache.find({src0p}); +} + +inline CacheBinary::iterator find(CacheBinary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p) { + return cache.find({src0p, src1p}); +} + +inline CacheTernary::iterator find(CacheTernary& cache, AstNodeDType*, DfgVertex* src0p, + DfgVertex* src1p, DfgVertex* src2p) { + return cache.find({src0p, src1p, src2p}); +} + +// These set the operands of a new vertex +inline void setOperands(DfgSel* vtxp, DfgVertex* fromp, uint32_t lsb) { + vtxp->fromp(fromp); + vtxp->lsb(lsb); +} + +inline void setOperands(DfgVertexUnary* vtxp, DfgVertex* src0p) { // + vtxp->relinkSource<0>(src0p); +} + +inline void setOperands(DfgVertexBinary* vtxp, DfgVertex* src0p, DfgVertex* src1p) { + vtxp->relinkSource<0>(src0p); + vtxp->relinkSource<1>(src1p); +} + +inline void setOperands(DfgVertexTernary* vtxp, DfgVertex* src0p, DfgVertex* src1p, + DfgVertex* src2p) { + vtxp->relinkSource<0>(src0p); + vtxp->relinkSource<1>(src1p); + vtxp->relinkSource<2>(src2p); +} + +// Get or create (and insert) vertex with given operands +template +inline Vertex* getOrCreate(DfgGraph& dfg, FileLine* flp, AstNodeDType* dtypep, Cache& cache, + Operands... operands) { + typename Cache::mapped_type& entrypr = getEntry(cache, dtypep, operands...); + if (!entrypr) { + Vertex* const newp = new Vertex{dfg, flp, dtypep}; + setOperands(newp, operands...); + entrypr = newp; + } + return reinterpret_cast(entrypr); +} + +// These add an existing vertex to the table, if an equivalent does not yet exist +inline void cache(CacheSel& cache, DfgSel* vtxp) { + DfgSel*& entrypr = getEntry(cache, vtxp->dtypep(), vtxp->fromp(), vtxp->lsb()); + if (!entrypr) entrypr = vtxp; +} + +inline void cache(CacheUnary& cache, DfgVertexUnary* vtxp) { + DfgVertexUnary*& entrypr = getEntry(cache, vtxp->dtypep(), vtxp->source<0>()); + if (!entrypr) entrypr = vtxp; +} + +inline void cache(CacheBinary& cache, DfgVertexBinary* vtxp) { + DfgVertexBinary*& entrypr + = getEntry(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>()); + if (!entrypr) entrypr = vtxp; +} + +inline void cache(CacheTernary& cache, DfgVertexTernary* vtxp) { + DfgVertexTernary*& entrypr + = getEntry(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>(), vtxp->source<2>()); + if (!entrypr) entrypr = vtxp; +} + +// These remove an existing vertex from the cache, if it is the cached vertex +inline void invalidateByValue(CacheSel& cache, DfgSel* vtxp) { + const auto it = find(cache, vtxp->dtypep(), vtxp->fromp(), vtxp->lsb()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +inline void invalidateByValue(CacheUnary& cache, DfgVertexUnary* vtxp) { + const auto it = find(cache, vtxp->dtypep(), vtxp->source<0>()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +inline void invalidateByValue(CacheBinary& cache, DfgVertexBinary* vtxp) { + const auto it = find(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +inline void invalidateByValue(CacheTernary& cache, DfgVertexTernary* vtxp) { + const auto it + = find(cache, vtxp->dtypep(), vtxp->source<0>(), vtxp->source<1>(), vtxp->source<2>()); + if (it != cache.end() && it->second == vtxp) cache.erase(it); +} + +// clang-format off + +// This type defines the cache type corresponding to a vertex type +template struct CacheTypeImpl : public CacheTypeImpl {}; +template <> struct CacheTypeImpl { using type = CacheSel; }; +template <> struct CacheTypeImpl { using type = CacheUnary; }; +template <> struct CacheTypeImpl { using type = CacheBinary; }; +template <> struct CacheTypeImpl { using type = CacheTernary; }; +template using CacheType = typename CacheTypeImpl::type; + +// Just add new lines in here for new vertex types you need to be able to cache + +#define FOREACH_CACHED_VERTEX_TYPE(macro) \ + macro(DfgSel) \ + macro(DfgNot) \ + macro(DfgNegate) \ + macro(DfgConcat) \ + macro(DfgNeq) \ + macro(DfgShiftL) \ + macro(DfgShiftR) \ + macro(DfgShiftRS) \ + macro(DfgAdd) \ + macro(DfgSub) \ + macro(DfgMul) \ + macro(DfgMulS) \ + macro(DfgEq) \ + macro(DfgAnd) \ + macro(DfgOr) \ + macro(DfgXor) \ + macro(DfgRedAnd) \ + macro(DfgRedOr) \ + macro(DfgRedXor) \ + macro(DfgCond) + +// clang-format on + +class V3DfgCache final { + DfgGraph& m_dfg; // The DfgGraph we are caching the vertices of + + // The per type caches +#define VERTEX_CACHE_DECLARE_LUT(t) CacheType m_cache##t; + FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_DECLARE_LUT) +#undef VERTEX_CACHE_DECLARE_LUT + + // Specializations return one of the above m_cache members + template + inline CacheType& cacheForType(); + +public: + V3DfgCache(DfgGraph& dfg) + : m_dfg{dfg} {} + + // Find a vertex of type 'Vertex', with the given operands, or create a new one and add it. + template + inline Vertex* getOrCreate(FileLine* flp, AstNodeDType* dtypep, Operands... operands); + + // Add an existing vertex of the table. If an equivalent already exists, then nothing happens. + void cache(DfgVertex* vtxp); + + // Remove an exiting vertex, it is the cached vertex. + void invalidateByValue(DfgVertex* vtxp); +}; + +// clang-format off +// The per-type specializations of 'V3DfgCache::cacheForType' +#define VERTEX_CACHE_DEFINE_LUT_SPECIALIZATION(t) \ + template <> inline CacheType& V3DfgCache::cacheForType() { return m_cache ## t; } +FOREACH_CACHED_VERTEX_TYPE(VERTEX_CACHE_DEFINE_LUT_SPECIALIZATION) +#undef VERTEX_CACHE_DEFINE_LUT_SPECIALIZATION +// clang-format on + +// Find a vertex of type 'Vertex', with the given operands, or create a new one and add it +template +Vertex* V3DfgCache::getOrCreate(FileLine* flp, AstNodeDType* dtypep, Operands... operands) { + static_assert(std::is_final::value, "Must invoke on final vertex type"); + constexpr bool isSel = std::is_same::value; + constexpr bool isUnary = !isSel && std::is_base_of::value; + constexpr bool isBinary = std::is_base_of::value; + constexpr bool isTernary = std::is_base_of::value; + static_assert(isSel || isUnary || isBinary || isTernary, + "'get' called with unknown vertex type"); + + static_assert(!isSel || sizeof...(Operands) == 2, // + "Wrong number of operands to DfgSel"); + static_assert(!isUnary || sizeof...(Operands) == 1, + "Wrong number of operands to unary vertex"); + static_assert(!isBinary || sizeof...(Operands) == 2, + "Wrong number of operands to binary vertex"); + static_assert(!isTernary || sizeof...(Operands) == 3, + "Wrong number of operands to ternary vertex"); + + return V3DfgCacheInternal::getOrCreate, Operands...>( + m_dfg, flp, dtypep, cacheForType(), operands...); +} + +} // namespace V3DfgCacheInternal + +// Export only the public interface class +using V3DfgCacheInternal::V3DfgCache; + +#endif // VERILATOR_V3DFGCACHE_H_ diff --git a/src/V3DfgPeephole.cpp b/src/V3DfgPeephole.cpp index e6fed4218..7b9ae8fbd 100644 --- a/src/V3DfgPeephole.cpp +++ b/src/V3DfgPeephole.cpp @@ -26,6 +26,7 @@ #include "V3DfgPeephole.h" #include "V3Dfg.h" +#include "V3DfgCache.h" #include "V3DfgPasses.h" #include "V3Stats.h" @@ -135,7 +136,6 @@ class V3DfgPeephole final : public DfgVisitor { // STATE DfgGraph& m_dfg; // The DfgGraph being visited V3DfgPeepholeContext& m_ctx; // The config structure - // bool m_changed = false; // Changed a vertex AstNodeDType* const m_bitDType = DfgVertex::dtypeForWidth(1); // Common, so grab it up front // Head of work list. Note that we want all next pointers in the list to be non-zero (including // that of the last element). This allows as to do two important things: detect if an element @@ -144,6 +144,9 @@ class V3DfgPeephole final : public DfgVisitor { // can easily check for the end of the list. DfgVertex* m_workListp = reinterpret_cast(this); + // Vertex lookup-table to avoid creating redundant vertices + V3DfgCache m_cache{m_dfg}; + #define APPLYING(id) if (checkApplying(VDfgPeepholePattern::id)) // METHODS @@ -151,7 +154,6 @@ class V3DfgPeephole final : public DfgVisitor { if (!m_ctx.m_enabled[id]) return false; UINFO(9, "Applying DFG patten " << id.ascii() << endl); ++m_ctx.m_count[id]; - // m_changed = true; return true; } @@ -180,27 +182,33 @@ class V3DfgPeephole final : public DfgVisitor { // the work list), but it will be deleted when the work list is processed. if (vtxp->getUser()) return; // Otherwise we can delete it now. + // Remove from cache + m_cache.invalidateByValue(vtxp); + // Unlink source edges + vtxp->forEachSourceEdge([](DfgEdge& edge, size_t) { edge.unlinkSource(); }); + // Should not have sinks + UASSERT_OBJ(!vtxp->hasSinks(), vtxp, "Should not delete used vertex"); + // VL_DO_DANGLING(vtxp->unlinkDelete(m_dfg), vtxp); } void replace(DfgVertex* vtxp, DfgVertex* replacementp) { + UASSERT_OBJ(vtxp != replacementp, vtxp, + "Should not try to replace with a vertex with itself"); + UASSERT_OBJ(vtxp->width() == replacementp->width(), vtxp, + "Replacement vertex has different width"); // Add sinks of replaced vertex to the work list addSinksToWorkList(vtxp); // Add replacement to the work list addToWorkList(replacementp); // Replace vertex with the replacement + vtxp->forEachSink([&](DfgVertex& sink) { m_cache.invalidateByValue(&sink); }); vtxp->replaceWith(replacementp); + replacementp->forEachSink([&](DfgVertex& sink) { m_cache.cache(&sink); }); // Vertex is now unused, so delete it deleteVertex(vtxp); } - void modified(DfgVertex* vtxp) { - // Add sinks of modified vertex to the work list - addSinksToWorkList(vtxp); - // Add the modified vertex itself to the work list - addToWorkList(vtxp); - } - // Shorthand static AstNodeDType* dtypeForWidth(uint32_t width) { return DfgVertex::dtypeForWidth(width); } @@ -211,23 +219,35 @@ class V3DfgPeephole final : public DfgVisitor { DfgConst* makeZero(FileLine* flp, uint32_t width) { return new DfgConst{m_dfg, flp, width}; } // Create a new vertex of the given type - template - Vertex* make(FileLine* flp, Args&&... args) { - static_assert(std::is_final::value, "Must invoke on final class"); - static_assert(!std::is_same::value, "Use 'makeZero' instead"); - static_assert(!std::is_base_of::value, "Can't create variables"); - // Create the new vertex - Vertex* const vtxp = new Vertex{m_dfg, flp, std::forward(args)...}; + template + Vertex* make(FileLine* flp, AstNodeDType* dtypep, Operands... operands) { + // Find or create an equivalent vertex + Vertex* const vtxp = m_cache.getOrCreate(flp, dtypep, operands...); + // Add to work list. - vtxp->template setUser(m_workListp); - m_workListp = vtxp; + DfgVertex*& workListNextp = vtxp->template user(); + if (!workListNextp) { + workListNextp = m_workListp; + m_workListp = vtxp; + } + // Return new node return vtxp; } + // Same as above, but 'flp' and 'dtypep' are taken from the given example vertex + template + Vertex* make(DfgVertex* examplep, Operands... operands) { + return make(examplep->fileline(), examplep->dtypep(), operands...); + } + + // Note: If any of the following transformers return true, then the vertex was replaced and the + // caller must not do any further changes, so the caller must check the return value, otherwise + // there will be hard to debug issues. + // Constant fold unary vertex, return true if folded template - bool foldUnary(Vertex* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool foldUnary(Vertex* vtxp) { static_assert(std::is_base_of::value, "Must invoke on unary"); static_assert(std::is_final::value, "Must invoke on final class"); if (DfgConst* const srcp = vtxp->srcp()->template cast()) { @@ -243,7 +263,7 @@ class V3DfgPeephole final : public DfgVisitor { // Constant fold binary vertex, return true if folded template - bool foldBinary(Vertex* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool foldBinary(Vertex* vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); if (DfgConst* const lhsp = vtxp->lhsp()->template cast()) { @@ -259,29 +279,10 @@ class V3DfgPeephole final : public DfgVisitor { return false; } - // Rotate the expression tree rooted at 'vtxp' to the right ('vtxp->lhsp()' becomes root, - // producing a right-leaning tree). Warning: only valid for associative operations. - template - void rotateRight(Vertex* vtxp) { - static_assert(std::is_base_of::value, "Must invoke on binary"); - static_assert(std::is_final::value, "Must invoke on final class"); - DfgVertexBinary* const ap = vtxp; - DfgVertexBinary* const bp = vtxp->lhsp()->template as(); - UASSERT_OBJ(!bp->hasMultipleSinks(), vtxp, "Can't rotate a non-tree"); - ap->replaceWith(bp); - ap->lhsp(bp->rhsp()); - bp->rhsp(ap); - // Concatenation dtypes need to be fixed up, other associative nodes preserve types - if VL_CONSTEXPR_CXX17 (std::is_same::value) { - ap->dtypep(dtypeForWidth(ap->lhsp()->width() + ap->rhsp()->width())); - bp->dtypep(dtypeForWidth(bp->lhsp()->width() + bp->rhsp()->width())); - } - } - // Transformations that apply to all associative binary vertices. // Returns true if vtxp was replaced. template - bool associativeBinary(Vertex* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool associativeBinary(Vertex* vtxp) { static_assert(std::is_base_of::value, "Must invoke on binary"); static_assert(std::is_final::value, "Must invoke on final class"); @@ -313,18 +314,9 @@ class V3DfgPeephole final : public DfgVisitor { foldOp(constp->num(), lConstp->num(), rlConstp->num()); // Replace vertex - if (!rVtxp->hasMultipleSinks()) { - rVtxp->lhsp(constp); - rVtxp->dtypep(vtxp->dtypep()); - replace(vtxp, rVtxp); - return true; - } else { - Vertex* const resp = make(flp, vtxp->dtypep()); - resp->lhsp(constp); - resp->rhsp(rVtxp->rhsp()); - replace(vtxp, resp); - return true; - } + Vertex* const resp = make(vtxp, constp, rVtxp->rhsp()); + replace(vtxp, resp); + return true; } } } @@ -342,79 +334,93 @@ class V3DfgPeephole final : public DfgVisitor { foldOp(constp->num(), lrConstp->num(), rConstp->num()); // Replace vertex - if (!lVtxp->hasMultipleSinks()) { - lVtxp->rhsp(constp); - lVtxp->dtypep(vtxp->dtypep()); - replace(vtxp, lVtxp); - return true; - } else { - Vertex* const resp = make(flp, vtxp->dtypep()); - resp->lhsp(lVtxp->lhsp()); - resp->rhsp(constp); - replace(vtxp, resp); - return true; - } + Vertex* const resp = make(vtxp, lVtxp->lhsp(), constp); + replace(vtxp, resp); + return true; } } } } // Make associative trees right leaning to reduce pattern variations, and for better CSE - while (vtxp->lhsp()->template is() && !vtxp->lhsp()->hasMultipleSinks()) { + bool changed = false; + while (true) { + Vertex* const lhsp = vtxp->lhsp()->template cast(); + if (!lhsp || lhsp->hasMultipleSinks()) break; + APPLYING(RIGHT_LEANING_ASSOC) { - rotateRight(vtxp); - modified(vtxp); - continue; + // Rotate the expression tree rooted at 'vtxp' to the right, producing a + // right-leaning tree + DfgVertex* const ap = lhsp->lhsp(); + DfgVertex* const bp = lhsp->rhsp(); + DfgVertex* const cp = vtxp->rhsp(); + + AstNodeDType* const rootDtyptp = vtxp->dtypep(); + AstNodeDType* childDtyptp = vtxp->dtypep(); + // Concatenation dtypes need to be fixed up, other associative nodes preserve + // types + if VL_CONSTEXPR_CXX17 (std::is_same::value) { + childDtyptp = dtypeForWidth(bp->width() + cp->width()); + } + + Vertex* const childp = make(vtxp->fileline(), childDtyptp, bp, cp); + Vertex* const rootp = make(lhsp->fileline(), rootDtyptp, ap, childp); + replace(vtxp, rootp); + changed = true; + vtxp = rootp; } - break; } - return false; + return changed; } // Transformations that apply to all commutative binary vertices - void commutativeBinary(DfgVertexBinary* vtxp) { - DfgVertex* const lhsp = vtxp->source<0>(); - DfgVertex* const rhsp = vtxp->source<1>(); + template + VL_ATTR_WARN_UNUSED_RESULT bool commutativeBinary(Vertex* vtxp) { + static_assert(std::is_base_of::value, "Must invoke on binary"); + static_assert(std::is_final::value, "Must invoke on final class"); + + DfgVertex* const lhsp = vtxp->lhsp(); + DfgVertex* const rhsp = vtxp->rhsp(); // Ensure Const is on left-hand side to simplify other patterns - if (lhsp->is()) return; + if (lhsp->is()) return false; if (rhsp->is()) { APPLYING(SWAP_CONST_IN_COMMUTATIVE_BINARY) { - vtxp->lhsp(rhsp); - vtxp->rhsp(lhsp); - modified(vtxp); - return; + Vertex* const replacementp = make(vtxp, rhsp, lhsp); + replace(vtxp, replacementp); + return true; } } // Ensure Not is on the left-hand side to simplify other patterns - if (lhsp->is()) return; + if (lhsp->is()) return false; if (rhsp->is()) { APPLYING(SWAP_NOT_IN_COMMUTATIVE_BINARY) { - vtxp->lhsp(rhsp); - vtxp->rhsp(lhsp); - modified(vtxp); - return; + Vertex* const replacementp = make(vtxp, rhsp, lhsp); + replace(vtxp, replacementp); + return true; } } - // If both sides are variable references, order the side in some defined way. This allows - // CSE to later merge 'a op b' with 'b op a'. + // If both sides are variable references, order the side in some defined way. This + // allows CSE to later merge 'a op b' with 'b op a'. if (lhsp->is() && rhsp->is()) { AstVar* const lVarp = lhsp->as()->varp(); AstVar* const rVarp = rhsp->as()->varp(); if (lVarp->name() > rVarp->name()) { APPLYING(SWAP_VAR_IN_COMMUTATIVE_BINARY) { - vtxp->lhsp(rhsp); - vtxp->rhsp(lhsp); - modified(vtxp); - return; + Vertex* const replacementp = make(vtxp, rhsp, lhsp); + replace(vtxp, replacementp); + return true; } } } + + return false; } // Bitwise operation with one side Const, and the other side a Concat template - bool tryPushBitwiseOpThroughConcat(Vertex* vtxp, DfgConst* constp, DfgConcat* concatp) { + VL_ATTR_WARN_UNUSED_RESULT bool tryPushBitwiseOpThroughConcat(Vertex* vtxp, DfgConst* constp, + DfgConcat* concatp) { UASSERT_OBJ(constp->dtypep() == concatp->dtypep(), vtxp, "Mismatched widths"); FileLine* const flp = vtxp->fileline(); @@ -432,24 +438,17 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t rWidth = rDtypep->width(); // The new Lhs vertex - Vertex* const newLhsp = make(flp, lDtypep); DfgConst* const newLhsConstp = makeZero(constp->fileline(), lWidth); newLhsConstp->num().opSel(constp->num(), width - 1, rWidth); - newLhsp->lhsp(newLhsConstp); - newLhsp->rhsp(concatp->lhsp()); + Vertex* const newLhsp = make(flp, lDtypep, newLhsConstp, concatp->lhsp()); // The new Rhs vertex - Vertex* const newRhsp = make(flp, rDtypep); DfgConst* const newRhsConstp = makeZero(constp->fileline(), rWidth); newRhsConstp->num().opSel(constp->num(), rWidth - 1, 0); - newRhsp->lhsp(newRhsConstp); - newRhsp->rhsp(concatp->rhsp()); + Vertex* const newRhsp = make(flp, rDtypep, newRhsConstp, concatp->rhsp()); // The replacement Concat vertex - DfgConcat* const newConcat - = make(concatp->fileline(), concatp->dtypep()); - newConcat->lhsp(newLhsp); - newConcat->rhsp(newRhsp); + DfgConcat* const newConcat = make(concatp, newLhsp, newRhsp); // Replace this vertex replace(vtxp, newConcat); @@ -460,13 +459,14 @@ class V3DfgPeephole final : public DfgVisitor { } template - bool tryPushCompareOpThroughConcat(Vertex* vtxp, DfgConst* constp, DfgConcat* concatp) { + VL_ATTR_WARN_UNUSED_RESULT bool tryPushCompareOpThroughConcat(Vertex* vtxp, DfgConst* constp, + DfgConcat* concatp) { UASSERT_OBJ(constp->dtypep() == concatp->dtypep(), vtxp, "Mismatched widths"); FileLine* const flp = vtxp->fileline(); - // If at least one of the sides of the Concat is constant, then push the Vertex past the - // Concat + // If at least one of the sides of the Concat is constant, then push the Vertex past + // the Concat if (concatp->lhsp()->is() || concatp->rhsp()->is()) { APPLYING(PUSH_COMPARE_OP_THROUGH_CONCAT) { const uint32_t width = concatp->width(); @@ -474,29 +474,25 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t rWidth = concatp->rhsp()->width(); // The new Lhs vertex - Vertex* const newLhsp = make(flp, m_bitDType); DfgConst* const newLhsConstp = makeZero(constp->fileline(), lWidth); newLhsConstp->num().opSel(constp->num(), width - 1, rWidth); - newLhsp->lhsp(newLhsConstp); - newLhsp->rhsp(concatp->lhsp()); + Vertex* const newLhsp + = make(flp, m_bitDType, newLhsConstp, concatp->lhsp()); // The new Rhs vertex - Vertex* const newRhsp = make(flp, m_bitDType); DfgConst* const newRhsConstp = makeZero(constp->fileline(), rWidth); newRhsConstp->num().opSel(constp->num(), rWidth - 1, 0); - newRhsp->lhsp(newRhsConstp); - newRhsp->rhsp(concatp->rhsp()); + Vertex* const newRhsp + = make(flp, m_bitDType, newRhsConstp, concatp->rhsp()); // The replacement Vertex DfgVertexBinary* const replacementp = std::is_same::value - ? make(concatp->fileline(), m_bitDType) + ? make(concatp->fileline(), m_bitDType, newLhsp, newRhsp) : nullptr; UASSERT_OBJ(replacementp, vtxp, "Unhandled vertex type in 'tryPushCompareOpThroughConcat': " << vtxp->typeName()); - replacementp->relinkSource<0>(newLhsp); - replacementp->relinkSource<1>(newRhsp); // Replace this vertex replace(vtxp, replacementp); @@ -507,7 +503,7 @@ class V3DfgPeephole final : public DfgVisitor { } template - bool tryPushBitwiseOpThroughReductions(Bitwise* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool tryPushBitwiseOpThroughReductions(Bitwise* vtxp) { using Reduction = BitwiseToReduction; if (Reduction* const lRedp = vtxp->lhsp()->template cast()) { @@ -518,11 +514,8 @@ class V3DfgPeephole final : public DfgVisitor { && !lSrcp->hasMultipleSinks() && !rSrcp->hasMultipleSinks()) { APPLYING(PUSH_BITWISE_THROUGH_REDUCTION) { FileLine* const flp = vtxp->fileline(); - Bitwise* const bwp = make(flp, lSrcp->dtypep()); - bwp->lhsp(lSrcp); - bwp->rhsp(rSrcp); - Reduction* const redp = make(flp, m_bitDType); - redp->srcp(bwp); + Bitwise* const bwp = make(flp, lSrcp->dtypep(), lSrcp, rSrcp); + Reduction* const redp = make(flp, m_bitDType, bwp); replace(vtxp, redp); return true; } @@ -534,10 +527,10 @@ class V3DfgPeephole final : public DfgVisitor { } template - void optimizeReduction(Reduction* vtxp) { + VL_ATTR_WARN_UNUSED_RESULT bool optimizeReduction(Reduction* vtxp) { using Bitwise = ReductionToBitwise; - if (foldUnary(vtxp)) return; + if (foldUnary(vtxp)) return true; DfgVertex* const srcp = vtxp->srcp(); FileLine* const flp = vtxp->fileline(); @@ -546,7 +539,7 @@ class V3DfgPeephole final : public DfgVisitor { if (srcp->dtypep() == m_bitDType) { APPLYING(REMOVE_WIDTH_ONE_REDUCTION) { replace(vtxp, srcp); - return; + return true; } } @@ -554,22 +547,18 @@ class V3DfgPeephole final : public DfgVisitor { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_REDUCTION_THROUGH_COND_WITH_CONST_BRANCH) { // The new 'then' vertex - Reduction* const newThenp = make(flp, m_bitDType); - newThenp->srcp(condp->thenp()); + Reduction* const newThenp = make(flp, m_bitDType, condp->thenp()); // The new 'else' vertex - Reduction* const newElsep = make(flp, m_bitDType); - newElsep->srcp(condp->elsep()); + Reduction* const newElsep = make(flp, m_bitDType, condp->elsep()); // The replacement Cond vertex - DfgCond* const newCondp = make(condp->fileline(), m_bitDType); - newCondp->condp(condp->condp()); - newCondp->thenp(newThenp); - newCondp->elsep(newElsep); + DfgCond* const newCondp = make(condp->fileline(), m_bitDType, + condp->condp(), newThenp, newElsep); // Replace this vertex replace(vtxp, newCondp); - return; + return true; } } } @@ -578,31 +567,36 @@ class V3DfgPeephole final : public DfgVisitor { if (concatp->lhsp()->is() || concatp->rhsp()->is()) { APPLYING(PUSH_REDUCTION_THROUGH_CONCAT) { // Reduce the parts of the concatenation - Reduction* const lRedp = make(concatp->fileline(), m_bitDType); - lRedp->srcp(concatp->lhsp()); - Reduction* const rRedp = make(concatp->fileline(), m_bitDType); - rRedp->srcp(concatp->rhsp()); + Reduction* const lRedp + = make(concatp->fileline(), m_bitDType, concatp->lhsp()); + Reduction* const rRedp + = make(concatp->fileline(), m_bitDType, concatp->rhsp()); // Bitwise reduce the results - Bitwise* const replacementp = make(flp, m_bitDType); - replacementp->lhsp(lRedp); - replacementp->rhsp(rRedp); + Bitwise* const replacementp = make(flp, m_bitDType, lRedp, rRedp); replace(vtxp, replacementp); - return; + return true; } } } + + return false; } - void optimizeShiftRHS(DfgVertexBinary* vtxp) { - if (const DfgConcat* const concatp = vtxp->rhsp()->cast()) { + template + VL_ATTR_WARN_UNUSED_RESULT bool optimizeShiftRHS(Shift* vtxp) { + static_assert(std::is_base_of::value, "Must invoke on binary"); + static_assert(std::is_final::value, "Must invoke on final class"); + if (const DfgConcat* const concatp = vtxp->rhsp()->template cast()) { if (concatp->lhsp()->isZero()) { // Drop redundant zero extension APPLYING(REMOVE_REDUNDANT_ZEXT_ON_RHS_OF_SHIFT) { - vtxp->rhsp(concatp->rhsp()); - modified(vtxp); + Shift* const replacementp = make(vtxp, vtxp->lhsp(), concatp->rhsp()); + replace(vtxp, replacementp); + return true; } } } + return false; } // VISIT methods @@ -626,14 +620,14 @@ class V3DfgPeephole final : public DfgVisitor { if (foldUnary(vtxp)) return; - // Convert all Extend into Concat with zeros. This simplifies other patterns as they only - // need to handle Concat, which is more generic, and don't need special cases for + // Convert all Extend into Concat with zeros. This simplifies other patterns as they + // only need to handle Concat, which is more generic, and don't need special cases for // Extend. APPLYING(REPLACE_EXTEND) { - FileLine* const flp = vtxp->fileline(); - DfgConcat* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(makeZero(flp, vtxp->width() - vtxp->srcp()->width())); - replacementp->rhsp(vtxp->srcp()); + DfgConcat* const replacementp = make( + vtxp, // + makeZero(vtxp->fileline(), vtxp->width() - vtxp->srcp()->width()), // + vtxp->srcp()); replace(vtxp, replacementp); return; } @@ -668,18 +662,14 @@ class V3DfgPeephole final : public DfgVisitor { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_NOT_THROUGH_COND) { // The new 'then' vertex - DfgNot* const newThenp = make(vtxp->fileline(), vtxp->dtypep()); - newThenp->srcp(condp->thenp()); + DfgNot* const newThenp = make(vtxp, condp->thenp()); // The new 'else' vertex - DfgNot* const newElsep = make(vtxp->fileline(), vtxp->dtypep()); - newElsep->srcp(condp->elsep()); + DfgNot* const newElsep = make(vtxp, condp->elsep()); // The replacement Cond vertex - DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep()); - newCondp->condp(condp->condp()); - newCondp->thenp(newThenp); - newCondp->elsep(newElsep); + DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep(), + condp->condp(), newThenp, newElsep); // Replace this vertex replace(vtxp, newCondp); @@ -701,9 +691,8 @@ class V3DfgPeephole final : public DfgVisitor { // Not of Eq if (DfgEq* const eqp = vtxp->srcp()->cast()) { APPLYING(REPLACE_NOT_EQ) { - DfgNeq* const replacementp = make(eqp->fileline(), vtxp->dtypep()); - replacementp->lhsp(eqp->lhsp()); - replacementp->rhsp(eqp->rhsp()); + DfgNeq* const replacementp + = make(eqp->fileline(), vtxp->dtypep(), eqp->lhsp(), eqp->rhsp()); replace(vtxp, replacementp); return; } @@ -712,9 +701,8 @@ class V3DfgPeephole final : public DfgVisitor { // Not of Neq if (DfgNeq* const neqp = vtxp->srcp()->cast()) { APPLYING(REPLACE_NOT_NEQ) { - DfgEq* const replacementp = make(neqp->fileline(), vtxp->dtypep()); - replacementp->lhsp(neqp->lhsp()); - replacementp->rhsp(neqp->rhsp()); + DfgEq* const replacementp = make(neqp->fileline(), vtxp->dtypep(), + neqp->lhsp(), neqp->rhsp()); replace(vtxp, replacementp); return; } @@ -730,11 +718,17 @@ class V3DfgPeephole final : public DfgVisitor { if (foldUnary(vtxp)) return; } - void visit(DfgRedOr* vtxp) override { optimizeReduction(vtxp); } + void visit(DfgRedOr* vtxp) override { + if (optimizeReduction(vtxp)) return; + } - void visit(DfgRedAnd* vtxp) override { optimizeReduction(vtxp); } + void visit(DfgRedAnd* vtxp) override { + if (optimizeReduction(vtxp)) return; + } - void visit(DfgRedXor* vtxp) override { optimizeReduction(vtxp); } + void visit(DfgRedXor* vtxp) override { + if (optimizeReduction(vtxp)) return; + } void visit(DfgSel* vtxp) override { DfgVertex* const fromp = vtxp->fromp(); @@ -771,15 +765,14 @@ class V3DfgPeephole final : public DfgVisitor { if (msb < rhsp->width()) { // If the select is entirely from rhs, then replace with sel from rhs APPLYING(REMOVE_SEL_FROM_RHS_OF_CONCAT) { // - vtxp->fromp(rhsp); - modified(vtxp); + DfgSel* const replacementp = make(vtxp, rhsp, vtxp->lsb()); + replace(vtxp, replacementp); } } else if (lsb >= rhsp->width()) { // If the select is entirely from the lhs, then replace with sel from lhs APPLYING(REMOVE_SEL_FROM_LHS_OF_CONCAT) { - vtxp->fromp(lhsp); - vtxp->lsb(lsb - rhsp->width()); - modified(vtxp); + DfgSel* const replacementp = make(vtxp, lhsp, lsb - rhsp->width()); + replace(vtxp, replacementp); } } else if (lsb == 0 || msb == concatp->width() - 1 // || lhsp->is() || rhsp->is() // @@ -792,20 +785,14 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t lSelWidth = width - rSelWidth; // The new Lhs vertex - DfgSel* const newLhsp = make(flp, dtypeForWidth(lSelWidth)); - newLhsp->fromp(lhsp); - newLhsp->lsb(0); + DfgSel* const newLhsp = make(flp, dtypeForWidth(lSelWidth), lhsp, 0U); // The new Rhs vertex - DfgSel* const newRhsp = make(flp, dtypeForWidth(rSelWidth)); - newRhsp->fromp(rhsp); - newRhsp->lsb(lsb); + DfgSel* const newRhsp = make(flp, dtypeForWidth(rSelWidth), rhsp, lsb); // The replacement Concat vertex DfgConcat* const newConcat - = make(concatp->fileline(), vtxp->dtypep()); - newConcat->lhsp(newLhsp); - newConcat->rhsp(newRhsp); + = make(concatp->fileline(), vtxp->dtypep(), newLhsp, newRhsp); // Replace this vertex replace(vtxp, newConcat); @@ -822,9 +809,8 @@ class V3DfgPeephole final : public DfgVisitor { const uint32_t newLsb = lsb % srcWidth; if (newLsb + width <= srcWidth) { APPLYING(PUSH_SEL_THROUGH_REPLICATE) { - vtxp->fromp(repp->srcp()); - vtxp->lsb(newLsb); - modified(vtxp); + DfgSel* const replacementp = make(vtxp, repp->srcp(), newLsb); + replace(vtxp, replacementp); } } } @@ -837,12 +823,11 @@ class V3DfgPeephole final : public DfgVisitor { UASSERT_OBJ(notp->srcp()->dtypep() == notp->dtypep(), notp, "Mismatched widths"); APPLYING(PUSH_SEL_THROUGH_NOT) { // Make Sel select from source of Not - vtxp->fromp(notp->srcp()); + DfgSel* const newSelp = make(vtxp, notp->srcp(), vtxp->lsb()); // Add Not after Sel - DfgNot* const replacementp = make(notp->fileline(), vtxp->dtypep()); - vtxp->replaceWith(replacementp); - replacementp->srcp(vtxp); - modified(vtxp); + DfgNot* const replacementp + = make(notp->fileline(), vtxp->dtypep(), newSelp); + replace(vtxp, replacementp); } } } @@ -850,11 +835,9 @@ class V3DfgPeephole final : public DfgVisitor { // Sel from Sel if (DfgSel* const selp = fromp->cast()) { APPLYING(REPLACE_SEL_FROM_SEL) { - // Make this Sel select from the source of the source Sel - vtxp->fromp(selp->fromp()); - // Adjust LSB - vtxp->lsb(lsb + selp->lsb()); - modified(vtxp); + // Make this Sel select from the source of the source Sel with adjusted LSB + DfgSel* const replacementp = make(vtxp, selp->fromp(), lsb + selp->lsb()); + replace(vtxp, replacementp); } } @@ -864,20 +847,14 @@ class V3DfgPeephole final : public DfgVisitor { if (condp->thenp()->is() || condp->elsep()->is()) { APPLYING(PUSH_SEL_THROUGH_COND) { // The new 'then' vertex - DfgSel* const newThenp = make(flp, vtxp->dtypep()); - newThenp->fromp(condp->thenp()); - newThenp->lsb(lsb); + DfgSel* const newThenp = make(vtxp, condp->thenp(), lsb); // The new 'else' vertex - DfgSel* const newElsep = make(flp, vtxp->dtypep()); - newElsep->fromp(condp->elsep()); - newElsep->lsb(lsb); + DfgSel* const newElsep = make(vtxp, condp->elsep(), lsb); // The replacement Cond vertex - DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep()); - newCondp->condp(condp->condp()); - newCondp->thenp(newThenp); - newCondp->elsep(newElsep); + DfgCond* const newCondp = make(condp->fileline(), vtxp->dtypep(), + condp->condp(), newThenp, newElsep); // Replace this vertex replace(vtxp, newCondp); @@ -892,13 +869,10 @@ class V3DfgPeephole final : public DfgVisitor { if (lsb == 0) { UASSERT_OBJ(shiftLp->lhsp()->width() >= width, vtxp, "input of shift narrow"); APPLYING(PUSH_SEL_THROUGH_SHIFTL) { - vtxp->fromp(shiftLp->lhsp()); - DfgShiftL* const newShiftLp - = make(shiftLp->fileline(), vtxp->dtypep()); - vtxp->replaceWith(newShiftLp); - newShiftLp->lhsp(vtxp); - newShiftLp->rhsp(shiftLp->rhsp()); - modified(vtxp); + DfgSel* const newSelp = make(vtxp, shiftLp->lhsp(), vtxp->lsb()); + DfgShiftL* const replacementp = make( + shiftLp->fileline(), vtxp->dtypep(), newSelp, shiftLp->rhsp()); + replace(vtxp, replacementp); } } } @@ -914,7 +888,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); @@ -925,25 +899,17 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NOT) { - DfgOr* const orp = make(flp, vtxp->dtypep()); - orp->lhsp(lhsNotp->srcp()); - orp->rhsp(rhsNotp->srcp()); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(orp); + DfgOr* const orp = make(vtxp, lhsNotp->srcp(), rhsNotp->srcp()); + DfgNot* const notp = make(vtxp, orp); replace(vtxp, notp); return; } } if (DfgNeq* const rhsNeqp = rhsp->cast()) { APPLYING(REPLACE_AND_OF_NOT_AND_NEQ) { - DfgOr* const orp = make(flp, vtxp->dtypep()); - orp->lhsp(lhsNotp->srcp()); - DfgEq* const newRhsp = make(rhsp->fileline(), rhsp->dtypep()); - newRhsp->lhsp(rhsNeqp->lhsp()); - newRhsp->rhsp(rhsNeqp->rhsp()); - orp->rhsp(newRhsp); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(orp); + DfgEq* const newRhsp = make(rhsp, rhsNeqp->lhsp(), rhsNeqp->rhsp()); + DfgOr* const orp = make(vtxp, lhsNotp->srcp(), newRhsp); + DfgNot* const notp = make(vtxp, orp); replace(vtxp, notp); return; } @@ -991,7 +957,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); @@ -1002,25 +968,17 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const lhsNotp = lhsp->cast()) { if (DfgNot* const rhsNotp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NOT) { - DfgAnd* const andp = make(flp, vtxp->dtypep()); - andp->lhsp(lhsNotp->srcp()); - andp->rhsp(rhsNotp->srcp()); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(andp); + DfgAnd* const andp = make(vtxp, lhsNotp->srcp(), rhsNotp->srcp()); + DfgNot* const notp = make(vtxp, andp); replace(vtxp, notp); return; } } if (DfgNeq* const rhsNeqp = rhsp->cast()) { APPLYING(REPLACE_OR_OF_NOT_AND_NEQ) { - DfgAnd* const andp = make(flp, vtxp->dtypep()); - andp->lhsp(lhsNotp->srcp()); - DfgEq* const newRhsp = make(rhsp->fileline(), rhsp->dtypep()); - newRhsp->lhsp(rhsNeqp->lhsp()); - newRhsp->rhsp(rhsNeqp->rhsp()); - andp->rhsp(newRhsp); - DfgNot* const notp = make(flp, vtxp->dtypep()); - notp->srcp(andp); + DfgEq* const newRhsp = make(rhsp, rhsNeqp->lhsp(), rhsNeqp->rhsp()); + DfgAnd* const andp = make(vtxp, lhsNotp->srcp(), newRhsp); + DfgNot* const notp = make(vtxp, andp); replace(vtxp, notp); return; } @@ -1033,18 +991,16 @@ class V3DfgPeephole final : public DfgVisitor { if (lhsConcatp->lhsp()->dtypep() == rhsConcatp->lhsp()->dtypep()) { if (lhsConcatp->lhsp()->isZero() && rhsConcatp->rhsp()->isZero()) { APPLYING(REPLACE_OR_OF_CONCAT_ZERO_LHS_AND_CONCAT_RHS_ZERO) { - DfgConcat* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(rhsConcatp->lhsp()); - replacementp->rhsp(lhsConcatp->rhsp()); + DfgConcat* const replacementp + = make(vtxp, rhsConcatp->lhsp(), lhsConcatp->rhsp()); replace(vtxp, replacementp); return; } } if (lhsConcatp->rhsp()->isZero() && rhsConcatp->lhsp()->isZero()) { APPLYING(REPLACE_OR_OF_CONCAT_LHS_ZERO_AND_CONCAT_ZERO_RHS) { - DfgConcat* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(lhsConcatp->lhsp()); - replacementp->rhsp(rhsConcatp->rhsp()); + DfgConcat* const replacementp + = make(vtxp, lhsConcatp->lhsp(), rhsConcatp->rhsp()); replace(vtxp, replacementp); return; } @@ -1094,11 +1050,10 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); - FileLine* const flp = vtxp->fileline(); if (DfgConst* const lConstp = lhsp->cast()) { if (lConstp->isZero()) { @@ -1109,14 +1064,13 @@ class V3DfgPeephole final : public DfgVisitor { } if (lConstp->isOnes()) { APPLYING(REPLACE_XOR_WITH_ONES) { - DfgNot* const replacementp = make(flp, vtxp->dtypep()); - replacementp->srcp(rhsp); + DfgNot* const replacementp = make(vtxp, rhsp); replace(vtxp, replacementp); return; } } if (DfgConcat* const rConcatp = rhsp->cast()) { - tryPushBitwiseOpThroughConcat(vtxp, lConstp, rConcatp); + if (tryPushBitwiseOpThroughConcat(vtxp, lConstp, rConcatp)) return; return; } } @@ -1134,7 +1088,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; } void visit(DfgArraySel* vtxp) override { @@ -1168,9 +1122,8 @@ class V3DfgPeephole final : public DfgVisitor { if (vtxp->dtypep() == rSelp->fromp()->dtypep() && rSelp->lsb() == lConstp->width()) { APPLYING(REPLACE_CONCAT_ZERO_AND_SEL_TOP_WITH_SHIFTR) { - DfgShiftR* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(rSelp->fromp()); - replacementp->rhsp(makeI32(flp, lConstp->width())); + DfgShiftR* const replacementp = make( + vtxp, rSelp->fromp(), makeI32(flp, lConstp->width())); replace(vtxp, replacementp); return; } @@ -1183,9 +1136,8 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgSel* const lSelp = lhsp->cast()) { if (vtxp->dtypep() == lSelp->fromp()->dtypep() && lSelp->lsb() == 0) { APPLYING(REPLACE_CONCAT_SEL_BOTTOM_AND_ZERO_WITH_SHIFTL) { - DfgShiftL* const replacementp = make(flp, vtxp->dtypep()); - replacementp->lhsp(lSelp->fromp()); - replacementp->rhsp(makeI32(flp, rConstp->width())); + DfgShiftL* const replacementp = make( + vtxp, lSelp->fromp(), makeI32(flp, rConstp->width())); replace(vtxp, replacementp); return; } @@ -1197,12 +1149,10 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const rNot = rhsp->cast()) { if (!lNot->hasMultipleSinks() && !rNot->hasMultipleSinks()) { APPLYING(PUSH_CONCAT_THROUGH_NOTS) { - vtxp->lhsp(lNot->srcp()); - vtxp->rhsp(rNot->srcp()); - DfgNot* const replacementp = make(flp, vtxp->dtypep()); - vtxp->replaceWith(replacementp); - replacementp->srcp(vtxp); - modified(vtxp); + DfgConcat* const newCatp + = make(vtxp, lNot->srcp(), rNot->srcp()); + DfgNot* const replacementp = make(vtxp, newCatp); + replace(vtxp, replacementp); return; } } @@ -1215,10 +1165,8 @@ class V3DfgPeephole final : public DfgVisitor { if (lSelp->lsb() == rSelp->lsb() + rSelp->width()) { // Two consecutive Sels, make a single Sel. const uint32_t width = lSelp->width() + rSelp->width(); - DfgSel* const joinedSelp = make(flp, dtypeForWidth(width)); - joinedSelp->fromp(rSelp->fromp()); - joinedSelp->lsb(rSelp->lsb()); - return joinedSelp; + return make(flp, dtypeForWidth(width), rSelp->fromp(), + rSelp->lsb()); } } return nullptr; @@ -1240,9 +1188,7 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgSel* const jointSelp = joinSels(lSelp, rlSelp, flp)) { APPLYING(REPLACE_NESTED_CONCAT_OF_ADJOINING_SELS_ON_LHS) { DfgConcat* const replacementp - = make(flp, vtxp->dtypep()); - replacementp->lhsp(jointSelp); - replacementp->rhsp(rConcatp->rhsp()); + = make(vtxp, jointSelp, rConcatp->rhsp()); replace(vtxp, replacementp); return; } @@ -1256,9 +1202,7 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgSel* const jointSelp = joinSels(lrlSelp, rSelp, flp)) { APPLYING(REPLACE_NESTED_CONCAT_OF_ADJOINING_SELS_ON_RHS) { DfgConcat* const replacementp - = make(flp, vtxp->dtypep()); - replacementp->lhsp(lConcatp->lhsp()); - replacementp->rhsp(jointSelp); + = make(vtxp, lConcatp->lhsp(), jointSelp); replace(vtxp, replacementp); return; } @@ -1280,7 +1224,7 @@ class V3DfgPeephole final : public DfgVisitor { void visit(DfgEq* vtxp) override { if (foldBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; DfgVertex* const lhsp = vtxp->lhsp(); DfgVertex* const rhsp = vtxp->rhsp(); @@ -1354,7 +1298,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; } void visit(DfgMulS* vtxp) override { @@ -1363,7 +1307,7 @@ class V3DfgPeephole final : public DfgVisitor { if (associativeBinary(vtxp)) return; - commutativeBinary(vtxp); + if (commutativeBinary(vtxp)) return; } void visit(DfgNeq* vtxp) override { @@ -1399,17 +1343,17 @@ class V3DfgPeephole final : public DfgVisitor { void visit(DfgShiftL* vtxp) override { if (foldBinary(vtxp)) return; - optimizeShiftRHS(vtxp); + if (optimizeShiftRHS(vtxp)) return; } void visit(DfgShiftR* vtxp) override { if (foldBinary(vtxp)) return; - optimizeShiftRHS(vtxp); + if (optimizeShiftRHS(vtxp)) return; } void visit(DfgShiftRS* vtxp) override { if (foldBinary(vtxp)) return; - optimizeShiftRHS(vtxp); + if (optimizeShiftRHS(vtxp)) return; } void visit(DfgSub* vtxp) override { @@ -1430,8 +1374,7 @@ class V3DfgPeephole final : public DfgVisitor { } if (vtxp->dtypep() == m_bitDType && rConstp->hasValue(1)) { APPLYING(REPLACE_SUB_WITH_NOT) { - DfgNot* const replacementp = make(vtxp->fileline(), m_bitDType); - replacementp->srcp(lhsp); + DfgNot* const replacementp = make(vtxp->fileline(), m_bitDType, lhsp); replace(vtxp, replacementp); return; } @@ -1471,10 +1414,9 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNot* const condNotp = condp->cast()) { if (!condp->hasMultipleSinks() || condNotp->hasMultipleSinks()) { APPLYING(SWAP_COND_WITH_NOT_CONDITION) { - vtxp->condp(condNotp->srcp()); - vtxp->thenp(elsep); - vtxp->elsep(thenp); - modified(vtxp); + DfgCond* const replacementp + = make(vtxp, condNotp->srcp(), elsep, thenp); + replace(vtxp, replacementp); return; } } @@ -1483,13 +1425,9 @@ class V3DfgPeephole final : public DfgVisitor { if (DfgNeq* const condNeqp = condp->cast()) { if (!condp->hasMultipleSinks()) { APPLYING(SWAP_COND_WITH_NEQ_CONDITION) { - DfgEq* const newCondp = make(condp->fileline(), condp->dtypep()); - newCondp->lhsp(condNeqp->lhsp()); - newCondp->rhsp(condNeqp->rhsp()); - vtxp->condp(newCondp); - vtxp->thenp(elsep); - vtxp->elsep(thenp); - modified(vtxp); + DfgEq* const newCondp = make(condp, condNeqp->lhsp(), condNeqp->rhsp()); + DfgCond* const replacementp = make(vtxp, newCondp, elsep, thenp); + replace(vtxp, replacementp); return; } } @@ -1500,13 +1438,11 @@ class V3DfgPeephole final : public DfgVisitor { if (!thenNotp->srcp()->is() && !elseNotp->srcp()->is() && !thenNotp->hasMultipleSinks() && !elseNotp->hasMultipleSinks()) { APPLYING(PULL_NOTS_THROUGH_COND) { + DfgCond* const newCondp = make( + vtxp, vtxp->condp(), thenNotp->srcp(), elseNotp->srcp()); DfgNot* const replacementp - = make(thenp->fileline(), vtxp->dtypep()); - vtxp->thenp(thenNotp->srcp()); - vtxp->elsep(elseNotp->srcp()); - vtxp->replaceWith(replacementp); - replacementp->srcp(vtxp); - modified(vtxp); + = make(thenp->fileline(), vtxp->dtypep(), newCondp); + replace(vtxp, replacementp); return; } } @@ -1520,13 +1456,11 @@ class V3DfgPeephole final : public DfgVisitor { if (constp->hasValue(1)) { if (thenAddp->rhsp() == elsep) { APPLYING(REPLACE_COND_INC) { - DfgConcat* const extp = make(flp, vtxp->dtypep()); - extp->rhsp(condp); - extp->lhsp(makeZero(flp, vtxp->width() - 1)); + DfgConcat* const extp = make( + vtxp, makeZero(flp, vtxp->width() - 1), condp); FileLine* const thenFlp = thenAddp->fileline(); - DfgAdd* const addp = make(thenFlp, vtxp->dtypep()); - addp->lhsp(thenAddp->rhsp()); - addp->rhsp(extp); + DfgAdd* const addp = make(thenFlp, vtxp->dtypep(), + thenAddp->rhsp(), extp); replace(vtxp, addp); return; } @@ -1540,13 +1474,11 @@ class V3DfgPeephole final : public DfgVisitor { if (constp->hasValue(1)) { if (thenSubp->lhsp() == elsep) { APPLYING(REPLACE_COND_DEC) { - DfgConcat* const extp = make(flp, vtxp->dtypep()); - extp->rhsp(condp); - extp->lhsp(makeZero(flp, vtxp->width() - 1)); + DfgConcat* const extp = make( + vtxp, makeZero(flp, vtxp->width() - 1), condp); FileLine* const thenFlp = thenSubp->fileline(); - DfgSub* const subp = make(thenFlp, vtxp->dtypep()); - subp->lhsp(thenSubp->lhsp()); - subp->rhsp(extp); + DfgSub* const subp = make(thenFlp, vtxp->dtypep(), + thenSubp->lhsp(), extp); replace(vtxp, subp); return; } @@ -1557,43 +1489,32 @@ class V3DfgPeephole final : public DfgVisitor { } if (vtxp->dtypep() == m_bitDType) { - AstNodeDType* const dtypep = vtxp->dtypep(); if (thenp->isZero()) { // a ? 0 : b becomes ~a & b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ZERO) { - DfgAnd* const repalcementp = make(flp, dtypep); - DfgNot* const notp = make(flp, dtypep); - notp->srcp(condp); - repalcementp->lhsp(notp); - repalcementp->rhsp(elsep); + DfgNot* const notp = make(vtxp, condp); + DfgAnd* const repalcementp = make(vtxp, notp, elsep); replace(vtxp, repalcementp); return; } } if (thenp->isOnes()) { // a ? 1 : b becomes a | b APPLYING(REPLACE_COND_WITH_THEN_BRANCH_ONES) { - DfgOr* const repalcementp = make(flp, dtypep); - repalcementp->lhsp(condp); - repalcementp->rhsp(elsep); + DfgOr* const repalcementp = make(vtxp, condp, elsep); replace(vtxp, repalcementp); return; } } if (elsep->isZero()) { // a ? b : 0 becomes a & b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_ZERO) { - DfgAnd* const repalcementp = make(flp, dtypep); - repalcementp->lhsp(condp); - repalcementp->rhsp(thenp); + DfgAnd* const repalcementp = make(vtxp, condp, thenp); replace(vtxp, repalcementp); return; } } if (elsep->isOnes()) { // a ? b : 1 becomes ~a | b APPLYING(REPLACE_COND_WITH_ELSE_BRANCH_ONES) { - DfgOr* const repalcementp = make(flp, dtypep); - DfgNot* const notp = make(flp, dtypep); - notp->srcp(condp); - repalcementp->lhsp(notp); - repalcementp->rhsp(thenp); + DfgNot* const notp = make(vtxp, condp); + DfgOr* const repalcementp = make(vtxp, notp, thenp); replace(vtxp, repalcementp); return; } @@ -1610,12 +1531,14 @@ class V3DfgPeephole final : public DfgVisitor { // DfgVertex::user is the next pointer of the work list elements const auto userDataInUse = m_dfg.userDataInUse(); - // Add all vertices to the work list. This also allocates all DfgVertex::user. + // Add all vertices to the work list, and to the vertex cache. + // This also allocates all DfgVertex::user. for (DfgVertex *vtxp = m_dfg.opVerticesBeginp(), *nextp; vtxp; vtxp = nextp) { nextp = vtxp->verticesNext(); if (VL_LIKELY(nextp)) VL_PREFETCH_RW(nextp); vtxp->setUser(m_workListp); m_workListp = vtxp; + m_cache.cache(vtxp); } // Process the work list diff --git a/src/astgen b/src/astgen index 1fa0ca3ca..942dc6747 100755 --- a/src/astgen +++ b/src/astgen @@ -1192,9 +1192,11 @@ def write_dfg_macros(filename): if node.isLeaf: emitBlock('''\ static constexpr VDfgType dfgType() {{ return VDfgType::at{t}; }}; + using Super = Dfg{s}; void accept(DfgVisitor& v) override {{ v.visit(this); }} ''', - t=node.name) + t=node.name, + s=node.superClass.name) for n in range(1, node.arity + 1): name, _, _ = node.getOp(n) diff --git a/test_regress/t/t_dfg_peephole.v b/test_regress/t/t_dfg_peephole.v index 7f0ea086e..b6745ef93 100644 --- a/test_regress/t/t_dfg_peephole.v +++ b/test_regress/t/t_dfg_peephole.v @@ -118,7 +118,7 @@ module t ( `signal(PUSH_NOT_THROUGH_COND, ~(rand_a[0] ? rand_a[4:0] : 5'hb)); `signal(REMOVE_NOT_NOT, ~~rand_a); `signal(REPLACE_NOT_NEQ, ~(rand_a != rand_b)); - `signal(REPLACE_NOT_EQ, ~(rand_a == rand_b)); + `signal(REPLACE_NOT_EQ, ~(srand_a == srand_b)); `signal(REPLACE_NOT_OF_CONST, ~4'd0); `signal(REPLACE_AND_OF_NOT_AND_NOT, ~rand_a[1] & ~rand_b[1]); `signal(REPLACE_AND_OF_NOT_AND_NEQ, ~rand_a[2] & (rand_b != 64'd2));