1 files changed, 19 insertions, 11 deletions

diff --git a/src/mpmc_ptr_ring.h b/src/mpmc_ptr_ring.h
index 258a16f..077e45e 100644
--- a/src/mpmc_ptr_ring.h
+++ b/src/mpmc_ptr_ring.h
@@ -78,21 +78,29 @@ static inline bool mpmc_ptr_ring_empty(struct mpmc_ptr_ring *r)
 
 static inline int mpmc_ptr_ring_produce(struct mpmc_ptr_ring *r, void *ptr)
 {
-	unsigned int p, c;
+	int p, c;
 	unsigned int mask = r->mask;
 
 	p = atomic_read(&r->producer_head);
 
 	for (;;) {
-		smp_rmb();	 /* TODO */
+		/*
+		 * The snapshot of producer must be up to date with respect to
+		 * consumer.
+		 */
+		smp_rmb();
 		c = atomic_read(&r->consumer_head);
 
-		if ((p - c) < mask) { /* fast path */
-			if (atomic_cmpxchg(&r->producer_head, p, p + 1) == p)
+		if (likely((p - c) < mask)) {
+			if (atomic_try_cmpxchg_relaxed(&r->producer_head, &p, p + 1))
 				break;
 		} else {
 			unsigned int new_p;
 
+			/*
+			 * Either the buffer is full or our copy of
+			 * producer_head is stale.
+			 */
 			smp_rmb();
 			new_p = atomic_read(&r->producer_head);
 
@@ -121,7 +129,7 @@ static inline int mpmc_ptr_ring_produce(struct mpmc_ptr_ring *r, void *ptr)
 
 static inline void *mpmc_ptr_ring_consume(struct mpmc_ptr_ring *r)
 {
-	unsigned int c, p, old_c;
+	int c, p;
 	unsigned int mask = r->mask;
 	void *element;
 
@@ -134,16 +142,16 @@ static inline void *mpmc_ptr_ring_consume(struct mpmc_ptr_ring *r)
 		p = atomic_read(&r->producer_tail);
 
 		/* Is the ring empty? */
-		if (p == c)
+		if (unlikely(p == c))
 			return NULL;
 
 		element = READ_ONCE(r->queue[c & mask]);
 
-		/* TODO: Why? */
-		smp_rmb();
-
-		old_c = atomic_cmpxchg(&r->consumer_head, c, c + 1);
-		if (old_c == c)
+		/*
+		 * Stores to consumer head must be completed before we update
+		 * the head, so we use *_release.
+		 */
+		if (atomic_try_cmpxchg_release(&r->consumer_head, &c, c + 1))
 			break;
 	}