make: add test-snap/test-diff targets (#4635)

.gitignore

@@ -43,3 +43,4 @@ verilator-config-version.cmake
 /.vscode/
 /.idea/
 /cmake-build-*/
+/test_regress/snapshot/

Makefile.in

@@ -173,6 +173,10 @@ smoke-test: all_nomsg
 test_regress: all_nomsg
 	$(MAKE) -C test_regress
 
+.PHONY: test-snap test-diff
+test-snap test-diff:
+	$(MAKE) -C test_regress $@
+
 examples: all_nomsg
 	for p in $(EXAMPLES) ; do \
 	  $(MAKE) -C $$p VERILATOR_ROOT=`pwd` || exit 10; \

docs/internals.rst

@@ -1462,6 +1462,19 @@ There are some traps to avoid when running regression tests
   system-dependent, but on Fedora Linux it would require editing the
   ``/etc/security/limits.conf`` file as root.
 
+Diffing generated code after changes
+------------------------------------
+
+When making a change in the code generation area that should not change the
+actual emitted code, it is useful to perform a diff to make sure the emitted
+code really did not change. To do this, the top level Makefile provides the
+*test-snap* and *test-diff* targets:
+
+- Run the test suite with ``make test``
+- Take a snapshot with ``make test-snap``
+- Apply your changes
+- Run the test suite again with ``make test``
+- See the changes in the output with ``make test-diff``
 
 Continuous Integration
 ----------------------

test_regress/Makefile

@@ -89,3 +89,45 @@ maintainer-copy::
 clean mostlyclean distclean maintainer-clean::
 	-rm -rf obj_* simv* simx* csrc cov_work INCA_libs *.log *.key logs vc_hdrs.h
 	-rm -rf t/obj_*
+
+distclean::
+	-rm -rf snapshot
+
+######################################################################
+# Generated code snapshot and diff for tests
+
+# Can be overridden for multiple snapshots
+TEST_SNAP_DIR ?= snapshot
+
+# Command to diff directories
+TEST_DIFF_TOOL ?= $(if $(shell which icdiff), icdiff -N -r, diff -r)
+
+TEST_SNAP_IGNORE := *.status *.log *.dat *.d *.o *.a *.so *stats.txt *.html \
+                    *.includecache *.out *.fst *.fst.vcd  *.tree *.dot \
+                    *.csv *.xml *.hash *.cmake gmon.out.* CMakeFiles \
+                    profile_exec.vcd t_pgo_threads
+
+define TEST_SNAP_template
+mkdir -p $(TEST_SNAP_DIR)
+rm -rf $(TEST_SNAP_DIR)/obj_$(1)
+cp -r obj_$(1) $(TEST_SNAP_DIR)/
+find $(TEST_SNAP_DIR)/obj_$(1) \( $(TEST_SNAP_IGNORE:%=-name "%" -o) \
+  -type f -executable \) -prune | xargs rm -r
+endef
+
+.PHONY: test-snap
+test-snap:
+	$(call TEST_SNAP_template,vlt)
+	$(call TEST_SNAP_template,vltmt)
+	$(call TEST_SNAP_template,dist)
+
+.PHONY: impl-test-diff
+impl-test-diff:
+	$(TEST_DIFF_TOOL) $(TEST_SNAP_DIR)/obj_vlt   obj_vlt
+	$(TEST_DIFF_TOOL) $(TEST_SNAP_DIR)/obj_vltmt obj_vltmt
+	$(TEST_DIFF_TOOL) $(TEST_SNAP_DIR)/obj_dist  obj_dist
+
+.PHONY: test-diff
+test-diff:
+	$(MAKE) impl-test-diff | grep -v "Only in obj_" | \
+    $$(git config --default less --global --get core.pager)