global: the _bh variety of rcu helpers have been unified

7 files changed, 57 insertions, 12 deletions

diff --git a/src/allowedips.c b/src/allowedips.c
index bfb6020..592501e 100644
--- a/src/allowedips.c
+++ b/src/allowedips.c
@@ -112,7 +112,7 @@ static void walk_remove_by_peer(struct allowedips_node __rcu **top,
 				if (!node->bit[0] || !node->bit[1]) {
 					rcu_assign_pointer(*nptr, DEREF(
 					       &node->bit[!REF(node->bit[0])]));
-					call_rcu_bh(&node->rcu, node_free_rcu);
+					call_rcu(&node->rcu, node_free_rcu);
 					node = DEREF(nptr);
 				}
 			}
@@ -300,12 +300,12 @@ void wg_allowedips_free(struct allowedips *table, struct mutex *lock)
 	RCU_INIT_POINTER(table->root6, NULL);
 	if (rcu_access_pointer(old4)) {
 		root_remove_peer_lists(old4);
-		call_rcu_bh(&rcu_dereference_protected(old4,
+		call_rcu(&rcu_dereference_protected(old4,
 				lockdep_is_held(lock))->rcu, root_free_rcu);
 	}
 	if (rcu_access_pointer(old6)) {
 		root_remove_peer_lists(old6);
-		call_rcu_bh(&rcu_dereference_protected(old6,
+		call_rcu(&rcu_dereference_protected(old6,
 				lockdep_is_held(lock))->rcu, root_free_rcu);
 	}
 }
diff --git a/src/compat/compat.h b/src/compat/compat.h
index 2dcdbae..396a293 100644
--- a/src/compat/compat.h
+++ b/src/compat/compat.h
@@ -778,6 +778,39 @@ struct __kernel_timespec {
 #define skb_probe_transport_header(a) skb_probe_transport_header(a, 0)
 #endif
 
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
+/* Note that all intentional uses of the non-_bh variety need to explicitly
+ * undef these, conditionalized on COMPAT_CANNOT_DEPRECIATE_BH_RCU.
+ */
+#include <linux/rcupdate.h>
+static __always_inline void old_synchronize_rcu(void)
+{
+	synchronize_rcu();
+}
+static __always_inline void old_call_rcu(void *a, void *b)
+{
+	call_rcu(a, b);
+}
+static __always_inline void old_rcu_barrier(void)
+{
+	rcu_barrier();
+}
+#ifdef synchronize_rcu
+#undef synchronize_rcu
+#endif
+#ifdef call_rcu
+#undef call_rcu
+#endif
+#ifdef rcu_barrier
+#undef rcu_barrier
+#endif
+#define synchronize_rcu synchronize_rcu_bh
+#define call_rcu call_rcu_bh
+#define rcu_barrier rcu_barrier_bh
+#define COMPAT_CANNOT_DEPRECIATE_BH_RCU
+#endif
+
 /* https://github.com/ClangBuiltLinux/linux/issues/7 */
 #if defined( __clang__) && (!defined(CONFIG_CLANG_VERSION) || CONFIG_CLANG_VERSION < 80000)
 #include <linux/bug.h>
diff --git a/src/device.c b/src/device.c
index 2866dd9..779c415 100644
--- a/src/device.c
+++ b/src/device.c
@@ -94,7 +94,7 @@ static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
 		mutex_unlock(&wg->device_update_lock);
 	}
 	rtnl_unlock();
-	rcu_barrier_bh();
+	rcu_barrier();
 	return 0;
 }
 
@@ -244,7 +244,7 @@ static void wg_destruct(struct net_device *dev)
 	destroy_workqueue(wg->packet_crypt_wq);
 	wg_packet_queue_free(&wg->decrypt_queue, true);
 	wg_packet_queue_free(&wg->encrypt_queue, true);
-	rcu_barrier_bh(); /* Wait for all the peers to be actually freed. */
+	rcu_barrier(); /* Wait for all the peers to be actually freed. */
 	wg_ratelimiter_uninit();
 	memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
 	skb_queue_purge(&wg->incoming_handshakes);
@@ -468,5 +468,5 @@ void wg_device_uninit(void)
 #ifdef CONFIG_PM_SLEEP
 	unregister_pm_notifier(&pm_notifier);
 #endif
-	rcu_barrier_bh();
+	rcu_barrier();
 }
diff --git a/src/noise.c b/src/noise.c
index 4405125..2e05e27 100644
--- a/src/noise.c
+++ b/src/noise.c
@@ -132,7 +132,7 @@ static void keypair_free_kref(struct kref *kref)
 			    keypair->entry.peer->internal_id);
 	wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable,
 				  &keypair->entry);
-	call_rcu_bh(&keypair->rcu, keypair_free_rcu);
+	call_rcu(&keypair->rcu, keypair_free_rcu);
 }
 
 void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now)
diff --git a/src/peer.c b/src/peer.c
index 996f40b..0c7e942 100644
--- a/src/peer.c
+++ b/src/peer.c
@@ -99,7 +99,7 @@ static void peer_make_dead(struct wg_peer *peer)
 	/* Mark as dead, so that we don't allow jumping contexts after. */
 	WRITE_ONCE(peer->is_dead, true);
 
-	/* The caller must now synchronize_rcu_bh() for this to take effect. */
+	/* The caller must now synchronize_rcu() for this to take effect. */
 }
 
 static void peer_remove_after_dead(struct wg_peer *peer)
@@ -171,7 +171,7 @@ void wg_peer_remove(struct wg_peer *peer)
 	lockdep_assert_held(&peer->device->device_update_lock);
 
 	peer_make_dead(peer);
-	synchronize_rcu_bh();
+	synchronize_rcu();
 	peer_remove_after_dead(peer);
 }
 
@@ -189,7 +189,7 @@ void wg_peer_remove_all(struct wg_device *wg)
 		peer_make_dead(peer);
 		list_add_tail(&peer->peer_list, &dead_peers);
 	}
-	synchronize_rcu_bh();
+	synchronize_rcu();
 	list_for_each_entry_safe(peer, temp, &dead_peers, peer_list)
 		peer_remove_after_dead(peer);
 }
@@ -228,7 +228,7 @@ static void kref_release(struct kref *refcount)
 	wg_packet_purge_staged_packets(peer);
 
 	/* Free the memory used. */
-	call_rcu_bh(&peer->rcu, rcu_release);
+	call_rcu(&peer->rcu, rcu_release);
 }
 
 void wg_peer_put(struct wg_peer *peer)
diff --git a/src/ratelimiter.c b/src/ratelimiter.c
index b3b1d25..fd09190 100644
--- a/src/ratelimiter.c
+++ b/src/ratelimiter.c
@@ -3,6 +3,18 @@
  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
  */
 
+#ifdef COMPAT_CANNOT_DEPRECIATE_BH_RCU
+/* We normally alias all non-_bh functions to the _bh ones in the compat layer,
+ * but that's not appropriate here, where we actually do want non-_bh ones.
+ */
+#undef synchronize_rcu
+#define synchronize_rcu old_synchronize_rcu
+#undef call_rcu
+#define call_rcu old_call_rcu
+#undef rcu_barrier
+#define rcu_barrier old_rcu_barrier
+#endif
+
 #include "ratelimiter.h"
 #include <linux/siphash.h>
 #include <linux/mm.h>
diff --git a/src/socket.c b/src/socket.c
index 652d798..5a77b0c 100644
--- a/src/socket.c
+++ b/src/socket.c
@@ -426,7 +426,7 @@ void wg_socket_reinit(struct wg_device *wg, struct sock *new4,
 	if (new4)
 		wg->incoming_port = ntohs(inet_sk(new4)->inet_sport);
 	mutex_unlock(&wg->socket_update_lock);
-	synchronize_rcu_bh();
+	synchronize_rcu();
 	synchronize_net();
 	sock_free(old4);
 	sock_free(old6);