Index: sys/crypto/crypto.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/crypto/crypto.c,v
retrieving revision 1.83
diff -u -p -u -p -r1.83 crypto.c
--- sys/crypto/crypto.c	30 Jun 2021 12:21:02 -0000	1.83
+++ sys/crypto/crypto.c	21 Jul 2021 15:52:45 -0000
@@ -387,26 +387,35 @@ crypto_unregister(u_int32_t driverid, in
 int
 crypto_dispatch(struct cryptop *crp)
 {
-	struct taskq *tq = crypto_taskq;
-	int s;
+	int error = 0, lock = 1, s;
 	u_int32_t hid;
 
 	s = splvm();
 	hid = (crp->crp_sid >> 32) & 0xffffffff;
 	if (hid < crypto_drivers_num) {
 		if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_MPSAFE)
-			tq = crypto_taskq_mpsafe;
+			lock = 0;
 	}
 	splx(s);
 
-	if (tq && !(crp->crp_flags & CRYPTO_F_NOQUEUE)) {
+	/* XXXSMP crypto_invoke() is not MP safe */
+	lock = 1;
+
+	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
+		if (lock)
+			KERNEL_LOCK();
+		error = crypto_invoke(crp);
+		if (lock)
+			KERNEL_UNLOCK();
+	} else {
+		struct taskq *tq;
+
+		tq = lock ? crypto_taskq : crypto_taskq_mpsafe;
 		task_set(&crp->crp_task, (void (*))crypto_invoke, crp);
 		task_add(tq, &crp->crp_task);
-	} else {
-		crypto_invoke(crp);
 	}
 
-	return 0;
+	return error;
 }
 
 /*
@@ -424,6 +433,8 @@ crypto_invoke(struct cryptop *crp)
 	if (crp == NULL || crp->crp_callback == NULL)
 		return EINVAL;
 
+	KERNEL_ASSERT_LOCKED();
+
 	s = splvm();
 	if (crp->crp_ndesc < 1 || crypto_drivers == NULL) {
 		crp->crp_etype = EINVAL;
@@ -535,11 +546,16 @@ void
 crypto_done(struct cryptop *crp)
 {
 	crp->crp_flags |= CRYPTO_F_DONE;
+
 	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
 		/* not from the crypto queue, wakeup the userland process */
 		crp->crp_callback(crp);
 	} else {
+		struct taskq *tq;
+
+		tq = (crp->crp_flags & CRYPTO_F_MPSAFE) ?
+		    crypto_taskq_mpsafe : crypto_taskq;
 		task_set(&crp->crp_task, (void (*))crp->crp_callback, crp);
-		task_add(crypto_taskq, &crp->crp_task);
+		task_add(tq, &crp->crp_task);
 	}
 }
Index: sys/crypto/cryptodev.h
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/crypto/cryptodev.h,v
retrieving revision 1.73
diff -u -p -u -p -r1.73 cryptodev.h
--- sys/crypto/cryptodev.h	9 Jul 2021 20:43:28 -0000	1.73
+++ sys/crypto/cryptodev.h	21 Jul 2021 15:52:45 -0000
@@ -171,6 +171,7 @@ struct cryptop {
 
 #define CRYPTO_F_IMBUF	0x0001	/* Input/output are mbuf chains, otherwise contig */
 #define CRYPTO_F_IOV	0x0002	/* Input/output are uio */
+#define CRYPTO_F_MPSAFE	0x0004	/* Do not use kernel lock for callback */
 #define CRYPTO_F_NOQUEUE	0x0008	/* Don't use crypto queue/thread */
 #define CRYPTO_F_DONE	0x0010	/* request completed */
 
Index: sys/netinet/ip_ah.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_ah.c,v
retrieving revision 1.151
diff -u -p -u -p -r1.151 ip_ah.c
--- sys/netinet/ip_ah.c	18 Jul 2021 14:38:20 -0000	1.151
+++ sys/netinet/ip_ah.c	21 Jul 2021 15:52:45 -0000
@@ -684,7 +684,7 @@ ah_input(struct mbuf *m, struct tdb *tdb
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -699,9 +699,7 @@ ah_input(struct mbuf *m, struct tdb *tdb
 	memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
 	tc->tc_rpl = tdb->tdb_rpl;
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
@@ -1134,7 +1132,7 @@ ah_output(struct mbuf *m, struct tdb *td
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_output_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -1148,9 +1146,7 @@ ah_output(struct mbuf *m, struct tdb *td
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
Index: sys/netinet/ip_esp.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_esp.c,v
retrieving revision 1.169
diff -u -p -u -p -r1.169 ip_esp.c
--- sys/netinet/ip_esp.c	18 Jul 2021 14:38:20 -0000	1.169
+++ sys/netinet/ip_esp.c	21 Jul 2021 15:52:45 -0000
@@ -498,7 +498,7 @@ esp_input(struct mbuf *m, struct tdb *td
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -528,9 +528,7 @@ esp_input(struct mbuf *m, struct tdb *td
 			crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
 	}
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
@@ -983,7 +981,7 @@ esp_output(struct mbuf *m, struct tdb *t
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_output_cb;
 	crp->crp_opaque = (caddr_t)tc;
@@ -1015,9 +1013,7 @@ esp_output(struct mbuf *m, struct tdb *t
 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
 	}
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
Index: sys/netinet/ip_ipcomp.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_ipcomp.c,v
retrieving revision 1.71
diff -u -p -u -p -r1.71 ip_ipcomp.c
--- sys/netinet/ip_ipcomp.c	8 Jul 2021 21:07:19 -0000	1.71
+++ sys/netinet/ip_ipcomp.c	21 Jul 2021 15:52:45 -0000
@@ -174,7 +174,7 @@ ipcomp_input(struct mbuf *m, struct tdb 
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -188,9 +188,7 @@ ipcomp_input(struct mbuf *m, struct tdb 
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	tc->tc_dst = tdb->tdb_dst;
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 }
 
@@ -479,15 +477,13 @@ ipcomp_output(struct mbuf *m, struct tdb
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len;	/* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_output_cb;
 	crp->crp_opaque = (caddr_t)tc;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
Index: sys/netinet/ipsec_input.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ipsec_input.c,v
retrieving revision 1.175
diff -u -p -u -p -r1.175 ipsec_input.c
--- sys/netinet/ipsec_input.c	8 Jul 2021 15:13:14 -0000	1.175
+++ sys/netinet/ipsec_input.c	21 Jul 2021 15:52:45 -0000
@@ -380,21 +380,19 @@ ipsec_input_cb(struct cryptop *crp)
 	struct tdb *tdb = NULL;
 	int clen, error;
 
-	KERNEL_ASSERT_LOCKED();
+	NET_ASSERT_LOCKED();
 
 	if (m == NULL) {
 		DPRINTF("bogus returned buffer from crypto");
 		ipsecstat_inc(ipsec_crypto);
-		goto droponly;
+		goto drop;
 	}
 
-
-	NET_LOCK();
 	tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
 	if (tdb == NULL) {
 		DPRINTF("TDB is expired while in crypto");
 		ipsecstat_inc(ipsec_notdb);
-		goto baddone;
+		goto drop;
 	}
 
 	/* Check for crypto errors */
@@ -403,13 +401,16 @@ ipsec_input_cb(struct cryptop *crp)
 			/* Reset the session ID */
 			if (tdb->tdb_cryptoid != 0)
 				tdb->tdb_cryptoid = crp->crp_sid;
-			NET_UNLOCK();
-			crypto_dispatch(crp);
+			error = crypto_dispatch(crp);
+			if (error) {
+				DPRINTF("crypto dispatch error %d", error);
+				goto drop;
+			}
 			return;
 		}
 		DPRINTF("crypto error %d", crp->crp_etype);
 		ipsecstat_inc(ipsec_noxform);
-		goto baddone;
+		goto drop;
 	}
 
 	/* Length of data after processing */
@@ -433,16 +434,13 @@ ipsec_input_cb(struct cryptop *crp)
 		    __func__, tdb->tdb_sproto);
 	}
 
-	NET_UNLOCK();
 	if (error) {
 		ipsecstat_inc(ipsec_idrops);
 		tdb->tdb_idrops++;
 	}
 	return;
 
- baddone:
-	NET_UNLOCK();
- droponly:
+ drop:
 	ipsecstat_inc(ipsec_idrops);
 	if (tdb != NULL)
 		tdb->tdb_idrops++;
Index: sys/netinet/ipsec_output.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ipsec_output.c,v
retrieving revision 1.82
diff -u -p -u -p -r1.82 ipsec_output.c
--- sys/netinet/ipsec_output.c	8 Jul 2021 15:13:14 -0000	1.82
+++ sys/netinet/ipsec_output.c	21 Jul 2021 15:52:45 -0000
@@ -395,20 +395,19 @@ ipsec_output_cb(struct cryptop *crp)
 	struct tdb *tdb = NULL;
 	int error, ilen, olen;
 
-	KERNEL_ASSERT_LOCKED();
+	NET_ASSERT_LOCKED();
 
 	if (m == NULL) {
 		DPRINTF("bogus returned buffer from crypto");
 		ipsecstat_inc(ipsec_crypto);
-		goto droponly;
+		goto drop;
 	}
 
-	NET_LOCK();
 	tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
 	if (tdb == NULL) {
 		DPRINTF("TDB is expired while in crypto");
 		ipsecstat_inc(ipsec_notdb);
-		goto baddone;
+		goto drop;
 	}
 
 	/* Check for crypto errors. */
@@ -417,13 +416,16 @@ ipsec_output_cb(struct cryptop *crp)
 			/* Reset the session ID */
 			if (tdb->tdb_cryptoid != 0)
 				tdb->tdb_cryptoid = crp->crp_sid;
-			NET_UNLOCK();
-			crypto_dispatch(crp);
+			error = crypto_dispatch(crp);
+			if (error) {
+				DPRINTF("crypto dispatch error %d", error);
+				goto drop;
+			}
 			return;
 		}
 		DPRINTF("crypto error %d", crp->crp_etype);
 		ipsecstat_inc(ipsec_noxform);
-		goto baddone;
+		goto drop;
 	}
 
 	olen = crp->crp_olen;
@@ -447,16 +449,13 @@ ipsec_output_cb(struct cryptop *crp)
 		    __func__, tdb->tdb_sproto);
 	}
 
-	NET_UNLOCK();
 	if (error) {
 		ipsecstat_inc(ipsec_odrops);
 		tdb->tdb_odrops++;
 	}
 	return;
 
- baddone:
-	NET_UNLOCK();
- droponly:
+ drop:
 	if (tdb != NULL)
 		tdb->tdb_odrops++;
 	m_freem(m);
@@ -479,6 +478,8 @@ ipsp_process_done(struct mbuf *m, struct
 	struct tdb_ident *tdbi;
 	struct m_tag *mtag;
 	int roff, error;
+
+	NET_ASSERT_LOCKED();
 
 	tdb->tdb_last_used = gettime();