malloc_debug: fix multiplication overflow in debug_calloc

The over flow check for nmemb * bytes in debug_calloc is incorrect,
use the builtin overflow functions to check for multiplication and
addition overflow.

Change-Id: I3f1c13102621bc5380be1f69caa88dba2118f3cb
(cherry picked from commit 239838608dbe9917acddfe5a51d92350a4c8e135)

1 files changed, 6 insertions, 0 deletions

diff --git a/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp b/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp
index 28729e8ac..2503981b0 100644
--- a/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp
+++ b/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp
@@ -1290,6 +1290,12 @@ TEST_F(MallocDebugTest, overflow) {
   ASSERT_TRUE(pointer == nullptr);
   ASSERT_EQ(ENOMEM, errno);
 
+  const size_t size_t_bits = sizeof(size_t) * 8;
+  const size_t sqrt_size_t = 1ULL << (size_t_bits/2);
+  pointer = debug_calloc(sqrt_size_t + 1, sqrt_size_t);
+  ASSERT_TRUE(pointer == nullptr);
+  ASSERT_EQ(ENOMEM, errno);
+
   pointer = debug_realloc(nullptr, SIZE_MAX);
   ASSERT_TRUE(pointer == nullptr);
   ASSERT_EQ(ENOMEM, errno);