crypto: return an error code from mbuf crypt routines

This permits returning different error codes for different conditions.

Signed-off-by: John Baldwin <jhb@FreeBSD.org>

3 files changed, 18 insertions, 13 deletions

diff --git a/src/crypto.c b/src/crypto.c
index 8e00266..7316f2d 100644
--- a/src/crypto.c
+++ b/src/crypto.c
@@ -587,7 +587,7 @@ chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
 	return ret;
 }
 
-static inline bool
+static inline int
 chacha20poly1305_crypt_mbuf(struct mbuf *m0, uint64_t nonce,
 			    const uint8_t key[CHACHA20POLY1305_KEY_SIZE], bool encrypt)
 {
@@ -596,7 +596,7 @@ chacha20poly1305_crypt_mbuf(struct mbuf *m0, uint64_t nonce,
 	uint8_t *buf, mbuf_mac[POLY1305_MAC_SIZE];
 	size_t len, leftover = 0;
 	struct mbuf *m;
-	bool ret;
+	int ret;
 	union {
 		uint32_t stream[CHACHA20_BLOCK_WORDS];
 		uint8_t block0[POLY1305_KEY_SIZE];
@@ -606,7 +606,7 @@ chacha20poly1305_crypt_mbuf(struct mbuf *m0, uint64_t nonce,
 
 	if (!encrypt) {
 		if (m0->m_pkthdr.len < POLY1305_MAC_SIZE)
-			return false;
+			return EMSGSIZE;
 		m_copydata(m0, m0->m_pkthdr.len - POLY1305_MAC_SIZE, POLY1305_MAC_SIZE, mbuf_mac);
 		m_adj(m0, -POLY1305_MAC_SIZE);
 	}
@@ -655,9 +655,9 @@ chacha20poly1305_crypt_mbuf(struct mbuf *m0, uint64_t nonce,
 	poly1305_final(&poly1305_state, b.mac);
 
 	if (encrypt)
-		ret = m_append(m0, POLY1305_MAC_SIZE, b.mac);
+		ret = m_append(m0, POLY1305_MAC_SIZE, b.mac) ? 0 : ENOMEM;
 	else
-		ret = timingsafe_bcmp(b.mac, mbuf_mac, POLY1305_MAC_SIZE) == 0;
+		ret = timingsafe_bcmp(b.mac, mbuf_mac, POLY1305_MAC_SIZE) == 0 ? 0 : EBADMSG;
 
 	explicit_bzero(&chacha20_state, sizeof(chacha20_state));
 	explicit_bzero(&b, sizeof(b));
@@ -665,14 +665,14 @@ chacha20poly1305_crypt_mbuf(struct mbuf *m0, uint64_t nonce,
 	return ret;
 }
 
-bool
+int
 chacha20poly1305_encrypt_mbuf(struct mbuf *m, const uint64_t nonce,
 			      const uint8_t key[CHACHA20POLY1305_KEY_SIZE])
 {
 	return chacha20poly1305_crypt_mbuf(m, nonce, key, true);
 }
 
-bool
+int
 chacha20poly1305_decrypt_mbuf(struct mbuf *m, const uint64_t nonce,
 			      const uint8_t key[CHACHA20POLY1305_KEY_SIZE])
 {
diff --git a/src/crypto.h b/src/crypto.h
index b1a5f0e..ad06066 100644
--- a/src/crypto.h
+++ b/src/crypto.h
@@ -27,11 +27,11 @@ chacha20poly1305_decrypt(uint8_t *dst, const uint8_t *src, const size_t src_len,
 			 const uint64_t nonce,
 			 const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
 
-bool
+int
 chacha20poly1305_encrypt_mbuf(struct mbuf *, const uint64_t nonce,
 			      const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
 
-bool
+int
 chacha20poly1305_decrypt_mbuf(struct mbuf *, const uint64_t nonce,
 			      const uint8_t key[CHACHA20POLY1305_KEY_SIZE]);
 
diff --git a/src/wg_noise.c b/src/wg_noise.c
index 6848627..d166543 100644
--- a/src/wg_noise.c
+++ b/src/wg_noise.c
@@ -903,8 +903,11 @@ noise_keep_key_fresh_recv(struct noise_remote *r)
 int
 noise_keypair_encrypt(struct noise_keypair *kp, uint32_t *r_idx, uint64_t nonce, struct mbuf *m)
 {
-	if (chacha20poly1305_encrypt_mbuf(m, nonce, kp->kp_send) == 0)
-	       return (ENOMEM);
+	int ret;
+
+	ret = chacha20poly1305_encrypt_mbuf(m, nonce, kp->kp_send);
+	if (ret)
+		return (ret);
 
 	*r_idx = kp->kp_index.i_remote_index;
 	return (0);
@@ -914,6 +917,7 @@ int
 noise_keypair_decrypt(struct noise_keypair *kp, uint64_t nonce, struct mbuf *m)
 {
 	uint64_t cur_nonce;
+	int ret;
 
 #ifdef __LP64__
 	cur_nonce = ck_pr_load_64(&kp->kp_nonce_recv);
@@ -927,8 +931,9 @@ noise_keypair_decrypt(struct noise_keypair *kp, uint64_t nonce, struct mbuf *m)
 	    noise_timer_expired(kp->kp_birthdate, REJECT_AFTER_TIME, 0))
 		return (EINVAL);
 
-	if (chacha20poly1305_decrypt_mbuf(m, nonce, kp->kp_recv) == 0)
-		return (EINVAL);
+	ret = chacha20poly1305_decrypt_mbuf(m, nonce, kp->kp_recv);
+	if (ret)
+		return (ret);
 
 	return (0);
 }