Fix ubsan warning in byteio_unittest

webrtc/modules/rtp_rtcp/source/byte_io_unittest.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
 #include <limits>
 
 #include "testing/gtest/include/gtest/gtest.h"
 #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
 
 namespace webrtc {
 namespace {
 
 class ByteIoTest : public ::testing::Test {
  protected:
   ByteIoTest() {}
   virtual ~ByteIoTest() {}
 
   enum { kAlignments = sizeof(uint64_t) - 1 };
 
   // Method to create a test value that is not the same when byte reversed.
   template <typename T>
   T CreateTestValue(bool negative, uint8_t num_bytes) {
+    // Examples of output:
+    // T = int32_t, negative = false, num_bytes = 4: 0x00010203
+    // T = int32_t, negative = true, num_bytes = 4: 0xFFFEFDFC
+    // T = int32_t, negative = false, num_bytes = 3: 0x000102
+    // * T = int32_t, negative = true, num_bytes = 3: 0xFFFEFD
+
     T val = 0;
     for (uint8_t i = 0; i != num_bytes; ++i) {
       val = (val << 8) + (negative ? (0xFF - i) : (i + 1));
     }
-    if (negative && std::numeric_limits<T>::is_signed) {
-      val |= static_cast<T>(-1) << (8 * num_bytes);
+
+    // This loop will create a sign extend mask if num_bytes if necessary.
+    // For the last example (marked * above), we the number needs to be sign

[stefan-webrtc, 2016/02/26 10:11:19]

"the number needs to..."

[sprang_webrtc, 2016/02/26 10:22:28]

Done.

+    // extended to be a valid int32_t. The sign extend mask is 0xFF000000.
+    // Comments for each step with this example below.
+    if (std::numeric_limits<T>::is_signed && negative &&
+        num_bytes < sizeof(T)) {
+      // Start with mask = 0xFFFFFFFF.
+      T mask = static_cast<T>(-1);
+      // Create a temporary for the lowest byte (0x000000FF).
+      const T neg_byte = static_cast<T>(0xFF);
+      for (int i = 0; i < num_bytes; ++i) {
+        // And the inverse of the temporary and the mask:
+        // 0xFFFFFFFF & 0xFFFFFF00 = 0xFFFFFF00.
+        // 0xFFFFFF00 & 0xFFFF00FF = 0xFFFF0000.
+        // 0xFFFF0000 & 0xFF00FFFF = 0xFF000000.
+        mask &= ~(neg_byte << (i * 8));
+      }
+      // Add the sign extension mask to the actual value.
+      val |= mask;
     }
     return val;
   }
 
   // Populate byte buffer with value, in big endian format.
   template <typename T>
   void PopulateTestData(uint8_t* data, T value, int num_bytes, bool bigendian) {
     if (bigendian) {
       for (int i = 0; i < num_bytes; ++i) {
         data[i] = (value >> ((num_bytes - i - 1) * 8)) & 0xFF;
@@ skipping 88 matching lines @@
       sizeof(int16_t)>(true);
   TestWrite<int16_t, ByteWriter<int16_t>::WriteBigEndian,
       sizeof(int16_t)>(true);
 }
 
 TEST_F(ByteIoTest, Test24SBitBigEndian) {
   TestRead<int32_t, ByteReader<int32_t, 3>::ReadBigEndian, 3>(true);
   TestWrite<int32_t, ByteWriter<int32_t, 3>::WriteBigEndian, 3>(true);
 }
 
-// Disabled for UBSan: https://bugs.chromium.org/p/webrtc/issues/detail?id=5490
-#ifdef UNDEFINED_SANITIZER
-#define MAYBE_Test32SBitBigEndian DISABLED_Test32SBitBigEndian
-#else
-#define MAYBE_Test32SBitBigEndian Test32SBitBigEndian
-#endif
-TEST_F(ByteIoTest, MAYBE_Test32SBitBigEndian) {
+TEST_F(ByteIoTest, Test32SBitBigEndian) {
   TestRead<int32_t, ByteReader<int32_t>::ReadBigEndian,
       sizeof(int32_t)>(true);
   TestWrite<int32_t, ByteWriter<int32_t>::WriteBigEndian,
       sizeof(int32_t)>(true);
 }
 
-// Disabled for UBSan: https://bugs.chromium.org/p/webrtc/issues/detail?id=5490
-#ifdef UNDEFINED_SANITIZER
-#define MAYBE_Test64SBitBigEndian DISABLED_Test64SBitBigEndian
-#else
-#define MAYBE_Test64SBitBigEndian Test64SBitBigEndian
-#endif
-TEST_F(ByteIoTest, MAYBE_Test64SBitBigEndian) {
+TEST_F(ByteIoTest, Test64SBitBigEndian) {
   TestRead<int64_t, ByteReader<int64_t>::ReadBigEndian,
       sizeof(int64_t)>(true);
   TestWrite<int64_t, ByteWriter<int64_t>::WriteBigEndian,
       sizeof(int64_t)>(true);
 }
 
 TEST_F(ByteIoTest, Test16UBitLittleEndian) {
   TestRead<uint16_t, ByteReader<uint16_t>::ReadLittleEndian,
       sizeof(uint16_t)>(false);
   TestWrite<uint16_t, ByteWriter<uint16_t>::WriteLittleEndian,
@@ skipping 24 matching lines @@
       sizeof(int16_t)>(false);
   TestWrite<int16_t, ByteWriter<int16_t>::WriteLittleEndian,
       sizeof(int16_t)>(false);
 }
 
 TEST_F(ByteIoTest, Test24SBitLittleEndian) {
   TestRead<int32_t, ByteReader<int32_t, 3>::ReadLittleEndian, 3>(false);
   TestWrite<int32_t, ByteWriter<int32_t, 3>::WriteLittleEndian, 3>(false);
 }
 
-// Disabled for UBSan: https://bugs.chromium.org/p/webrtc/issues/detail?id=5490
-#ifdef UNDEFINED_SANITIZER
-#define MAYBE_Test32SBitLittleEndian DISABLED_Test32SBitLittleEndian
-#else
-#define MAYBE_Test32SBitLittleEndian Test32SBitLittleEndian
-#endif
-TEST_F(ByteIoTest, MAYBE_Test32SBitLittleEndian) {
+TEST_F(ByteIoTest, Test32SBitLittleEndian) {
   TestRead<int32_t, ByteReader<int32_t>::ReadLittleEndian,
       sizeof(int32_t)>(false);
   TestWrite<int32_t, ByteWriter<int32_t>::WriteLittleEndian,
       sizeof(int32_t)>(false);
 }
 
-// Disabled for UBSan: https://bugs.chromium.org/p/webrtc/issues/detail?id=5490
-#ifdef UNDEFINED_SANITIZER
-#define MAYBE_Test64SBitLittleEndian DISABLED_Test64SBitLittleEndian
-#else
-#define MAYBE_Test64SBitLittleEndian Test64SBitLittleEndian
-#endif
-TEST_F(ByteIoTest, MAYBE_Test64SBitLittleEndian) {
+TEST_F(ByteIoTest, Test64SBitLittleEndian) {
   TestRead<int64_t, ByteReader<int64_t>::ReadLittleEndian,
       sizeof(int64_t)>(false);
   TestWrite<int64_t, ByteWriter<int64_t>::WriteLittleEndian,
       sizeof(int64_t)>(false);
 }
 
 // Sets up a fixed byte array and converts N bytes from the array into a
 // uint64_t. Verifies the value with hard-coded reference.
 TEST(ByteIo, SanityCheckFixedByteArrayUnsignedReadBigEndian) {
   uint8_t data[8] = {0xFF, 0xEE, 0xDD, 0xCC, 0xBB, 0xAA, 0x99, 0x88};
@@ skipping 26 matching lines @@
   EXPECT_EQ(static_cast<uint64_t>(0xBBCCDDEEFF), value);
   value = ByteReader<uint64_t, 6>::ReadLittleEndian(data);
   EXPECT_EQ(static_cast<uint64_t>(0xAABBCCDDEEFF), value);
   value = ByteReader<uint64_t, 7>::ReadLittleEndian(data);
   EXPECT_EQ(static_cast<uint64_t>(0x99AABBCCDDEEFF), value);
   value = ByteReader<uint64_t, 8>::ReadLittleEndian(data);
   EXPECT_EQ(static_cast<uint64_t>(0x8899AABBCCDDEEFF), value);
 }
 }  // namespace
 }  // namespace webrtc