[NFC][sanitizer] Remove SetSoftRssLimitExceededCallback

According comments on D44404, something like that was the goal.

Reviewed By: morehouse, kstoimenov

Differential Revision: https://reviews.llvm.org/D114991

Author: vitalybuka

compiler-rt/lib/asan/asan_allocator.cpp

@@ -305,7 +305,6 @@ struct Allocator {
   QuarantineCache fallback_quarantine_cache;
 
   uptr max_user_defined_malloc_size;
-  atomic_uint8_t rss_limit_exceeded;
 
   // ------------------- Options --------------------------
   atomic_uint16_t min_redzone;
@@ -345,14 +344,6 @@ struct Allocator {
                                        : kMaxAllowedMallocSize;
   }
 
-  bool IsRssLimitExceeded() {
-    return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
-  }
-
-  void SetRssLimitExceeded(bool limit_exceeded) {
-    atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
-  }
-
   void RePoisonChunk(uptr chunk) {
     // This could be a user-facing chunk (with redzones), or some internal
     // housekeeping chunk, like TransferBatch. Start by assuming the former.
@@ -1071,10 +1062,6 @@ void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS {
   instance.ForceUnlock();
 }
 
-void AsanSoftRssLimitExceededCallback(bool limit_exceeded) {
-  instance.SetRssLimitExceeded(limit_exceeded);
-}
-
 }  // namespace __asan
 
 // --- Implementation of LSan-specific functions --- {{{1

compiler-rt/lib/asan/asan_rtl.cpp

@@ -450,8 +450,6 @@ static void AsanInitInternal() {
   allocator_options.SetFrom(flags(), common_flags());
   InitializeAllocator(allocator_options);
 
-  SetSoftRssLimitExceededCallback(AsanSoftRssLimitExceededCallback);
-
   // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited
   // should be set to 1 prior to initializing the threads.
   asan_inited = 1;

compiler-rt/lib/memprof/memprof_allocator.cpp

@@ -218,7 +218,6 @@ struct Allocator {
   AllocatorCache fallback_allocator_cache;
 
   uptr max_user_defined_malloc_size;
-  atomic_uint8_t rss_limit_exceeded;
 
   // Holds the mapping of stack ids to MemInfoBlocks.
   MIBMapTy MIBMap;
@@ -301,14 +300,6 @@ struct Allocator {
                                        : kMaxAllowedMallocSize;
   }
 
-  bool IsRssLimitExceeded() {
-    return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
-  }
-
-  void SetRssLimitExceeded(bool limit_exceeded) {
-    atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
-  }
-
   // -------------------- Allocation/Deallocation routines ---------------
   void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack,
                  AllocType alloc_type) {
@@ -662,10 +653,6 @@ uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp) {
   return usable_size;
 }
 
-void MemprofSoftRssLimitExceededCallback(bool limit_exceeded) {
-  instance.SetRssLimitExceeded(limit_exceeded);
-}
-
 } // namespace __memprof
 
 // ---------------------- Interface ---------------- {{{1

compiler-rt/lib/memprof/memprof_allocator.h

@@ -98,7 +98,6 @@ int memprof_posix_memalign(void **memptr, uptr alignment, uptr size,
 uptr memprof_malloc_usable_size(const void *ptr, uptr pc, uptr bp);
 
 void PrintInternalAllocatorStats();
-void MemprofSoftRssLimitExceededCallback(bool exceeded);
 
 } // namespace __memprof
 #endif // MEMPROF_ALLOCATOR_H

compiler-rt/lib/memprof/memprof_rtl.cpp

@@ -135,8 +135,6 @@ void PrintAddressSpaceLayout() {
 
 static bool UNUSED __local_memprof_dyninit = [] {
   MaybeStartBackgroudThread();
-  SetSoftRssLimitExceededCallback(MemprofSoftRssLimitExceededCallback);
-
   return false;
 }();
 

compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp

@@ -195,4 +195,14 @@ void PrintHintAllocatorCannotReturnNull() {
          "allocator_may_return_null=1\n");
 }
 
+static atomic_uint8_t rss_limit_exceeded;
+
+bool IsRssLimitExceeded() {
+  return atomic_load(&rss_limit_exceeded, memory_order_relaxed);
+}
+
+void SetRssLimitExceeded(bool limit_exceeded) {
+  atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);
+}
+
 } // namespace __sanitizer

compiler-rt/lib/sanitizer_common/sanitizer_allocator.h

@@ -70,6 +70,9 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
 #include "sanitizer_allocator_secondary.h"
 #include "sanitizer_allocator_combined.h"
 
+bool IsRssLimitExceeded();
+void SetRssLimitExceeded(bool limit_exceeded);
+
 } // namespace __sanitizer
 
 #endif // SANITIZER_ALLOCATOR_H

compiler-rt/lib/sanitizer_common/sanitizer_common.h

@@ -326,12 +326,6 @@ void SetUserDieCallback(DieCallbackType callback);
 
 void SetCheckUnwindCallback(void (*callback)());
 
-// Callback will be called if soft_rss_limit_mb is given and the limit is
-// exceeded (exceeded==true) or if rss went down below the limit
-// (exceeded==false).
-// The callback should be registered once at the tool init time.
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded));
-
 // Functions related to signal handling.
 typedef void (*SignalHandlerType)(int, void *, void *);
 HandleSignalMode GetHandleSignalMode(int signum);

compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp

@@ -10,6 +10,7 @@
 // run-time libraries.
 //===----------------------------------------------------------------------===//
 
+#include "sanitizer_allocator.h"
 #include "sanitizer_allocator_interface.h"
 #include "sanitizer_common.h"
 #include "sanitizer_flags.h"
@@ -18,12 +19,6 @@
 
 namespace __sanitizer {
 
-static void (*SoftRssLimitExceededCallback)(bool exceeded);
-void SetSoftRssLimitExceededCallback(void (*Callback)(bool exceeded)) {
-  CHECK_EQ(SoftRssLimitExceededCallback, nullptr);
-  SoftRssLimitExceededCallback = Callback;
-}
-
 #if (SANITIZER_LINUX || SANITIZER_NETBSD) && !SANITIZER_GO
 // Weak default implementation for when sanitizer_stackdepot is not linked in.
 SANITIZER_WEAK_ATTRIBUTE StackDepotStats StackDepotGetStats() { return {}; }
@@ -67,13 +62,11 @@ void *BackgroundThread(void *arg) {
         reached_soft_rss_limit = true;
         Report("%s: soft rss limit exhausted (%zdMb vs %zdMb)\n",
                SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
-        if (SoftRssLimitExceededCallback)
-          SoftRssLimitExceededCallback(true);
+        SetRssLimitExceeded(true);
       } else if (soft_rss_limit_mb >= current_rss_mb &&
                  reached_soft_rss_limit) {
         reached_soft_rss_limit = false;
-        if (SoftRssLimitExceededCallback)
-          SoftRssLimitExceededCallback(false);
+        SetRssLimitExceeded(false);
       }
     }
     if (heap_profile &&