build: add UndefinedBehaviorSanitizer support (#3870)

- add UBSAN build option

- turn on UBSAN CI build

Fixes: https://github.com/libuv/libuv/issues/3869

.github/workflows/sanitizer.yml

@@ -50,3 +50,12 @@ jobs:
         continue-on-error: true # currently permit failures
         run: |
           ./build-tsan/uv_run_tests_a
+
+      - name: UBSAN Build
+        run: |
+          mkdir build-ubsan
+          (cd build-ubsan && cmake .. -G Ninja -DBUILD_TESTING=ON -DUBSAN=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang)
+          cmake --build build-ubsan
+      - name: UBSAN Test
+        run: |
+          ./build-ubsan/uv_run_tests_a

CMakeLists.txt

@@ -43,6 +43,7 @@ endif()
 option(ASAN "Enable AddressSanitizer (ASan)" OFF)
 option(MSAN "Enable MemorySanitizer (MSan)" OFF)
 option(TSAN "Enable ThreadSanitizer (TSan)" OFF)
+option(UBSAN "Enable UndefinedBehaviorSanitizer (UBSan)" OFF)
 
 if(MSAN AND NOT CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang")
   message(SEND_ERROR "MemorySanitizer requires clang. Try again with -DCMAKE_C_COMPILER=clang")
@@ -79,6 +80,19 @@ if(TSAN)
   endif()
 endif()
 
+if(UBSAN)
+  list(APPEND uv_defines __UBSAN__=1)
+  if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|GNU|Clang")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined")
+    set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined")
+  elseif(MSVC)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /fsanitize=undefined")
+  else()
+    message(SEND_ERROR "UndefinedBehaviorSanitizer support requires clang, gcc, or msvc. Try again with -DCMAKE_C_COMPILER.")
+  endif()
+endif()
+
 # Compiler check
 string(CONCAT is-msvc $<OR:
   $<C_COMPILER_ID:MSVC>,