reduce memory order constraints for better efficiency on ARM etc

3 files changed, 21 insertions, 10 deletions

diff --git a/include/mimalloc-atomic.h b/include/mimalloc-atomic.h
index 30d1e4f..b9935cb 100644
--- a/include/mimalloc-atomic.h
+++ b/include/mimalloc-atomic.h
@@ -27,20 +27,24 @@ terms of the MIT license. A copy of the license can be found in the file
 // Atomic operations specialized for mimalloc
 // ------------------------------------------------------
 
-// Atomically add a value; returns the previous value. Memory ordering is acquire-release.
+// Atomically add a value; returns the previous value. Memory ordering is relaxed.
 static inline uintptr_t mi_atomic_add(_Atomic(uintptr_t)* p, uintptr_t add);
 
-// Atomically "and" a value; returns the previous value. Memory ordering is acquire-release.
+// Atomically "and" a value; returns the previous value. Memory ordering is release.
 static inline uintptr_t mi_atomic_and(_Atomic(uintptr_t)* p, uintptr_t x);
 
-// Atomically "or" a value; returns the previous value. Memory ordering is acquire-release.
+// Atomically "or" a value; returns the previous value. Memory ordering is release.
 static inline uintptr_t mi_atomic_or(_Atomic(uintptr_t)* p, uintptr_t x);
 
 // Atomically compare and exchange a value; returns `true` if successful.
-// May fail spuriously. Memory ordering is acquire-release; with acquire on failure.
+// May fail spuriously. Memory ordering is release; with relaxed on failure.
 static inline bool mi_atomic_cas_weak(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired);
 
 // Atomically compare and exchange a value; returns `true` if successful.
+// May fail spuriously. Memory ordering is acquire-release; with acquire on failure.
+static inline bool mi_atomic_cas_weak_acq_rel(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired);
+
+// Atomically compare and exchange a value; returns `true` if successful.
 // Memory ordering is acquire-release; with acquire on failure.
 static inline bool mi_atomic_cas_strong(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired);
 
@@ -180,6 +184,9 @@ static inline bool mi_atomic_cas_strong(_Atomic(uintptr_t)* p, uintptr_t* expect
 static inline bool mi_atomic_cas_weak(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired) {
   return mi_atomic_cas_strong(p,expected,desired);
 }
+static inline bool mi_atomic_cas_weak_acq_rel(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired) {
+  return mi_atomic_cas_strong(p, expected, desired);
+}
 static inline uintptr_t mi_atomic_exchange(_Atomic(uintptr_t)* p, uintptr_t exchange) {
   return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
 }
@@ -225,18 +232,22 @@ static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t
 #endif
 static inline uintptr_t mi_atomic_add(_Atomic(uintptr_t)* p, uintptr_t add) {
   MI_USING_STD
-  return atomic_fetch_add_explicit(p, add, memory_order_acq_rel);
+  return atomic_fetch_add_explicit(p, add, memory_order_relaxed);
 }
 static inline uintptr_t mi_atomic_and(_Atomic(uintptr_t)* p, uintptr_t x) {
   MI_USING_STD
-  return atomic_fetch_and_explicit(p, x, memory_order_acq_rel);
+  return atomic_fetch_and_explicit(p, x, memory_order_release);
 }
 static inline uintptr_t mi_atomic_or(_Atomic(uintptr_t)* p, uintptr_t x) {
   MI_USING_STD
-  return atomic_fetch_or_explicit(p, x, memory_order_acq_rel);
+  return atomic_fetch_or_explicit(p, x, memory_order_release);
 }
 static inline bool mi_atomic_cas_weak(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired) {
   MI_USING_STD
+  return atomic_compare_exchange_weak_explicit(p, expected, desired, memory_order_release, memory_order_relaxed);
+}
+static inline bool mi_atomic_cas_weak_acq_rel(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired) {
+  MI_USING_STD
   return atomic_compare_exchange_weak_explicit(p, expected, desired, memory_order_acq_rel, memory_order_acquire);
 }
 static inline bool mi_atomic_cas_strong(_Atomic(uintptr_t)* p, uintptr_t* expected, uintptr_t desired) {
@@ -266,7 +277,7 @@ static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add)
 static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
   MI_USING_STD
   int64_t current = atomic_load_explicit((_Atomic(int64_t)*)p, memory_order_relaxed);
-  while (current < x && !atomic_compare_exchange_weak_explicit((_Atomic(int64_t)*)p, &current, x, memory_order_acq_rel, memory_order_acquire)) { /* nothing */ };
+  while (current < x && !atomic_compare_exchange_weak_explicit((_Atomic(int64_t)*)p, &current, x, memory_order_release, memory_order_relaxed)) { /* nothing */ };
 }
 #endif
 
diff --git a/src/page.c b/src/page.c
index 6b92d4c..92faf9f 100644
--- a/src/page.c
+++ b/src/page.c
@@ -159,7 +159,7 @@ static void _mi_page_thread_free_collect(mi_page_t* page)
   do {
     head = mi_tf_block(tfree);
     tfreex = mi_tf_set_block(tfree,NULL);
-  } while (!mi_atomic_cas_weak(&page->xthread_free, &tfree, tfreex));
+  } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tfree, tfreex));
 
   // return if the list is empty
   if (head == NULL) return;
diff --git a/src/segment.c b/src/segment.c
index 5af98b1..5523055 100644
--- a/src/segment.c
+++ b/src/segment.c
@@ -982,7 +982,7 @@ static mi_segment_t* mi_abandoned_pop(void) {
       mi_segment_t* anext = mi_atomic_read_ptr_relaxed(mi_segment_t, &segment->abandoned_next);
       next = mi_tagged_segment(anext, ts); // note: reads the segment's `abandoned_next` field so should not be decommitted
     }
-  } while (segment != NULL && !mi_atomic_cas_weak(&abandoned, &ts, next));
+  } while (segment != NULL && !mi_atomic_cas_weak_acq_rel(&abandoned, &ts, next));
   mi_atomic_decrement(&abandoned_readers);  // release reader lock
   if (segment != NULL) {
     mi_atomic_write_ptr(mi_segment_t, &segment->abandoned_next, NULL);