Index: sys/crypto/crypto.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/crypto/crypto.c,v
retrieving revision 1.83
diff -u -p -u -p -r1.83 crypto.c
--- sys/crypto/crypto.c	30 Jun 2021 12:21:02 -0000	1.83
+++ sys/crypto/crypto.c	16 Jul 2021 20:42:16 -0000
@@ -387,26 +387,33 @@ crypto_unregister(u_int32_t driverid, in
 int
 crypto_dispatch(struct cryptop *crp)
 {
-	struct taskq *tq = crypto_taskq;
-	int s;
+	int error = 0, lock = 1, s;
 	u_int32_t hid;
 
 	s = splvm();
 	hid = (crp->crp_sid >> 32) & 0xffffffff;
 	if (hid < crypto_drivers_num) {
 		if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_MPSAFE)
-			tq = crypto_taskq_mpsafe;
+			/* XXXSMP crypto_invoke() is not MP safe */
+			lock = 1;
 	}
 	splx(s);
 
-	if (tq && !(crp->crp_flags & CRYPTO_F_NOQUEUE)) {
+	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
+		if (lock)
+			KERNEL_LOCK();
+		error = crypto_invoke(crp);
+		if (lock)
+			KERNEL_UNLOCK();
+	} else {
+		struct taskq *tq;
+
+		tq = lock ? crypto_taskq : crypto_taskq_mpsafe;
 		task_set(&crp->crp_task, (void (*))crypto_invoke, crp);
 		task_add(tq, &crp->crp_task);
-	} else {
-		crypto_invoke(crp);
 	}
 
-	return 0;
+	return error;
 }
 
 /*
@@ -424,6 +431,8 @@ crypto_invoke(struct cryptop *crp)
 	if (crp == NULL || crp->crp_callback == NULL)
 		return EINVAL;
 
+	KERNEL_ASSERT_LOCKED();
+
 	s = splvm();
 	if (crp->crp_ndesc < 1 || crypto_drivers == NULL) {
 		crp->crp_etype = EINVAL;
@@ -534,12 +543,25 @@ crypto_init(void)
 void
 crypto_done(struct cryptop *crp)
 {
+	int lock = 1;
+
 	crp->crp_flags |= CRYPTO_F_DONE;
+
+	if (crp->crp_flags & CRYPTO_F_MPSAFE)
+		lock = 0;
+
 	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
 		/* not from the crypto queue, wakeup the userland process */
+		if (lock)
+			KERNEL_LOCK();
 		crp->crp_callback(crp);
+		if (lock)
+			KERNEL_UNLOCK();
 	} else {
+		struct taskq *tq;
+
+		tq = lock ? crypto_taskq : crypto_taskq_mpsafe;
 		task_set(&crp->crp_task, (void (*))crp->crp_callback, crp);
-		task_add(crypto_taskq, &crp->crp_task);
+		task_add(tq, &crp->crp_task);
 	}
 }
Index: sys/crypto/cryptodev.h
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/crypto/cryptodev.h,v
retrieving revision 1.73
diff -u -p -u -p -r1.73 cryptodev.h
--- sys/crypto/cryptodev.h	9 Jul 2021 20:43:28 -0000	1.73
+++ sys/crypto/cryptodev.h	16 Jul 2021 20:42:16 -0000
@@ -171,6 +171,7 @@ struct cryptop {
 
 #define CRYPTO_F_IMBUF	0x0001	/* Input/output are mbuf chains, otherwise contig */
 #define CRYPTO_F_IOV	0x0002	/* Input/output are uio */
+#define CRYPTO_F_MPSAFE	0x0004	/* Do not use kernel lock for callback */
 #define CRYPTO_F_NOQUEUE	0x0008	/* Don't use crypto queue/thread */
 #define CRYPTO_F_DONE	0x0010	/* request completed */
 
Index: sys/netinet/ip_ah.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_ah.c,v
retrieving revision 1.150
diff -u -p -u -p -r1.150 ip_ah.c
--- sys/netinet/ip_ah.c	8 Jul 2021 21:07:19 -0000	1.150
+++ sys/netinet/ip_ah.c	16 Jul 2021 20:42:16 -0000
@@ -550,7 +550,7 @@ ah_input(struct mbuf *m, struct tdb *tdb
 		    sizeof(u_int32_t), &btsx);
 		btsx = ntohl(btsx);
 
-		switch (checkreplaywindow(tdb, btsx, &esn, 0)) {
+		switch (checkreplaywindow(tdb, tdb->tdb_rpl, btsx, &esn, 0)) {
 		case 0: /* All's well. */
 			break;
 		case 1:
@@ -684,7 +684,7 @@ ah_input(struct mbuf *m, struct tdb *tdb
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -697,10 +697,9 @@ ah_input(struct mbuf *m, struct tdb *tdb
 	tc->tc_proto = tdb->tdb_sproto;
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
+	tc->tc_rpl = tdb->tdb_rpl;
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
@@ -715,6 +714,7 @@ ah_input_cb(struct tdb *tdb, struct tdb_
 {
 	const struct auth_hash *ahx = tdb->tdb_authalgxform;
 	int roff, rplen, skip, protoff;
+	u_int64_t rpl;
 	u_int32_t btsx, esn;
 	caddr_t ptr;
 	unsigned char calc[AH_ALEN_MAX];
@@ -727,6 +727,7 @@ ah_input_cb(struct tdb *tdb, struct tdb_
 
 	skip = tc->tc_skip;
 	protoff = tc->tc_protoff;
+	rpl = tc->tc_rpl;
 
 	rplen = AH_FLENGTH + sizeof(u_int32_t);
 
@@ -756,7 +757,7 @@ ah_input_cb(struct tdb *tdb, struct tdb_
 		    sizeof(u_int32_t), &btsx);
 		btsx = ntohl(btsx);
 
-		switch (checkreplaywindow(tdb, btsx, &esn, 1)) {
+		switch (checkreplaywindow(tdb, rpl, btsx, &esn, 1)) {
 		case 0: /* All's well. */
 #if NPFSYNC > 0
 			pfsync_update_tdb(tdb,0);
@@ -1131,7 +1132,7 @@ ah_output(struct mbuf *m, struct tdb *td
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_output_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -1145,9 +1146,7 @@ ah_output(struct mbuf *m, struct tdb *td
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
Index: sys/netinet/ip_esp.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_esp.c,v
retrieving revision 1.167
diff -u -p -u -p -r1.167 ip_esp.c
--- sys/netinet/ip_esp.c	8 Jul 2021 21:07:19 -0000	1.167
+++ sys/netinet/ip_esp.c	16 Jul 2021 20:42:16 -0000
@@ -388,7 +388,7 @@ esp_input(struct mbuf *m, struct tdb *td
 		    &btsx);
 		btsx = ntohl(btsx);
 
-		switch (checkreplaywindow(tdb, btsx, &esn, 0)) {
+		switch (checkreplaywindow(tdb, tdb->tdb_rpl, btsx, &esn, 0)) {
 		case 0: /* All's well */
 			break;
 		case 1:
@@ -498,7 +498,7 @@ esp_input(struct mbuf *m, struct tdb *td
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -511,6 +511,7 @@ esp_input(struct mbuf *m, struct tdb *td
 	tc->tc_proto = tdb->tdb_sproto;
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	tc->tc_dst = tdb->tdb_dst;
+	tc->tc_rpl = tdb->tdb_rpl;
 
 	/* Decryption descriptor */
 	if (espx) {
@@ -527,9 +528,7 @@ esp_input(struct mbuf *m, struct tdb *td
 			crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
 	}
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
@@ -549,6 +548,7 @@ esp_input_cb(struct tdb *tdb, struct tdb
 	int hlen, roff, skip, protoff;
 	struct mbuf *m1, *mo;
 	const struct auth_hash *esph;
+	u_int64_t rpl;
 	u_int32_t btsx, esn;
 	caddr_t ptr;
 #ifdef ENCDEBUG
@@ -557,6 +557,7 @@ esp_input_cb(struct tdb *tdb, struct tdb
 
 	skip = tc->tc_skip;
 	protoff = tc->tc_protoff;
+	rpl = tc->tc_rpl;
 
 	NET_ASSERT_LOCKED();
 
@@ -590,7 +591,7 @@ esp_input_cb(struct tdb *tdb, struct tdb
 		    &btsx);
 		btsx = ntohl(btsx);
 
-		switch (checkreplaywindow(tdb, btsx, &esn, 1)) {
+		switch (checkreplaywindow(tdb, rpl, btsx, &esn, 1)) {
 		case 0: /* All's well */
 #if NPFSYNC > 0
 			pfsync_update_tdb(tdb,0);
@@ -980,7 +981,7 @@ esp_output(struct mbuf *m, struct tdb *t
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_output_cb;
 	crp->crp_opaque = (caddr_t)tc;
@@ -1012,9 +1013,7 @@ esp_output(struct mbuf *m, struct tdb *t
 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
 	}
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
@@ -1049,14 +1048,15 @@ esp_output_cb(struct tdb *tdb, struct td
  * return 3 for packet within current window but already received
  */
 int
-checkreplaywindow(struct tdb *tdb, u_int32_t seq, u_int32_t *seqh, int commit)
+checkreplaywindow(struct tdb *tdb, u_int64_t t, u_int32_t seq, u_int32_t *seqh,
+    int commit)
 {
 	u_int32_t	tl, th, wl;
 	u_int32_t	packet, window = TDB_REPLAYMAX - TDB_REPLAYWASTE;
 	int		idx, esn = tdb->tdb_flags & TDBF_ESN;
 
-	tl = (u_int32_t)tdb->tdb_rpl;
-	th = (u_int32_t)(tdb->tdb_rpl >> 32);
+	tl = (u_int32_t)t;
+	th = (u_int32_t)(t >> 32);
 
 	/* Zero SN is not allowed */
 	if ((esn && seq == 0 && tl <= AH_HMAC_INITIAL_RPL && th == 0) ||
Index: sys/netinet/ip_ipcomp.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_ipcomp.c,v
retrieving revision 1.71
diff -u -p -u -p -r1.71 ip_ipcomp.c
--- sys/netinet/ip_ipcomp.c	8 Jul 2021 21:07:19 -0000	1.71
+++ sys/netinet/ip_ipcomp.c	16 Jul 2021 20:42:16 -0000
@@ -174,7 +174,7 @@ ipcomp_input(struct mbuf *m, struct tdb 
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
@@ -188,9 +188,7 @@ ipcomp_input(struct mbuf *m, struct tdb 
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	tc->tc_dst = tdb->tdb_dst;
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 }
 
@@ -479,15 +477,13 @@ ipcomp_output(struct mbuf *m, struct tdb
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len;	/* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF;
+	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_callback = ipsec_output_cb;
 	crp->crp_opaque = (caddr_t)tc;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	KERNEL_LOCK();
 	error = crypto_dispatch(crp);
-	KERNEL_UNLOCK();
 	return error;
 
  drop:
Index: sys/netinet/ip_ipsp.h
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ip_ipsp.h,v
retrieving revision 1.201
diff -u -p -u -p -r1.201 ip_ipsp.h
--- sys/netinet/ip_ipsp.h	13 Jul 2021 08:16:17 -0000	1.201
+++ sys/netinet/ip_ipsp.h	16 Jul 2021 20:42:16 -0000
@@ -432,12 +432,13 @@ struct tdb_ident {
 };
 
 struct tdb_crypto {
-	u_int32_t		tc_spi;
 	union sockaddr_union	tc_dst;
-	u_int8_t		tc_proto;
+	u_int64_t		tc_rpl;
+	u_int32_t		tc_spi;
 	int			tc_protoff;
 	int			tc_skip;
 	u_int			tc_rdomain;
+	u_int8_t		tc_proto;
 };
 
 struct ipsecinit {
@@ -622,7 +623,7 @@ int	tcp_signature_tdb_output(struct mbuf
 	  int, int);
 
 /* Replay window */
-int	checkreplaywindow(struct tdb *, u_int32_t, u_int32_t *, int);
+int	checkreplaywindow(struct tdb *, u_int64_t, u_int32_t, u_int32_t *, int);
 
 /* Packet processing */
 int	ipsp_process_packet(struct mbuf *, struct tdb *, int, int);
Index: sys/netinet/ipsec_input.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ipsec_input.c,v
retrieving revision 1.175
diff -u -p -u -p -r1.175 ipsec_input.c
--- sys/netinet/ipsec_input.c	8 Jul 2021 15:13:14 -0000	1.175
+++ sys/netinet/ipsec_input.c	16 Jul 2021 20:42:16 -0000
@@ -380,14 +380,11 @@ ipsec_input_cb(struct cryptop *crp)
 	struct tdb *tdb = NULL;
 	int clen, error;
 
-	KERNEL_ASSERT_LOCKED();
-
 	if (m == NULL) {
 		DPRINTF("bogus returned buffer from crypto");
 		ipsecstat_inc(ipsec_crypto);
 		goto droponly;
 	}
-
 
 	NET_LOCK();
 	tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
Index: sys/netinet/ipsec_output.c
===================================================================
RCS file: /mount/openbsd/cvs/src/sys/netinet/ipsec_output.c,v
retrieving revision 1.82
diff -u -p -u -p -r1.82 ipsec_output.c
--- sys/netinet/ipsec_output.c	8 Jul 2021 15:13:14 -0000	1.82
+++ sys/netinet/ipsec_output.c	16 Jul 2021 20:42:16 -0000
@@ -395,8 +395,6 @@ ipsec_output_cb(struct cryptop *crp)
 	struct tdb *tdb = NULL;
 	int error, ilen, olen;
 
-	KERNEL_ASSERT_LOCKED();
-
 	if (m == NULL) {
 		DPRINTF("bogus returned buffer from crypto");
 		ipsecstat_inc(ipsec_crypto);
@@ -418,7 +416,12 @@ ipsec_output_cb(struct cryptop *crp)
 			if (tdb->tdb_cryptoid != 0)
 				tdb->tdb_cryptoid = crp->crp_sid;
 			NET_UNLOCK();
-			crypto_dispatch(crp);
+			error = crypto_dispatch(crp);
+			if (error) {
+				DPRINTF("crypto dispatch error %d", error);
+				ipsecstat_inc(ipsec_odrops);
+				tdb->tdb_odrops++;
+			}
 			return;
 		}
 		DPRINTF("crypto error %d", crp->crp_etype);
@@ -479,6 +482,8 @@ ipsp_process_done(struct mbuf *m, struct
 	struct tdb_ident *tdbi;
 	struct m_tag *mtag;
 	int roff, error;
+
+	NET_ASSERT_LOCKED();
 
 	tdb->tdb_last_used = gettime();