Date: Tue, 8 Oct 2019 21:34:07 +0000 (UTC) From: John Baldwin <jhb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r353328 - in head/sys: kern netinet sys Message-ID: <201910082134.x98LY7YB088492@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jhb Date: Tue Oct 8 21:34:06 2019 New Revision: 353328 URL: https://svnweb.freebsd.org/changeset/base/353328 Log: Add a TOE KTLS mode and a TOE hook for allocating TLS sessions. This adds the glue to allocate TLS sessions and invokes it from the TLS enable socket option handler. This also adds some counters for active TOE sessions. The TOE KTLS mode is returned by getsockopt(TLSTX_TLS_MODE) when TOE KTLS is in use on a socket, but cannot be set via setsockopt(). To simplify various checks, a TLS session now includes an explicit 'mode' member set to the value returned by TLSTX_TLS_MODE. Various places that used to check 'sw_encrypt' against NULL to determine software vs ifnet (NIC) TLS now check 'mode' instead. Reviewed by: np, gallatin Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D21891 Modified: head/sys/kern/kern_sendfile.c head/sys/kern/uipc_ktls.c head/sys/kern/uipc_socket.c head/sys/netinet/tcp.h head/sys/netinet/tcp_offload.c head/sys/netinet/tcp_offload.h head/sys/netinet/tcp_usrreq.c head/sys/netinet/toecore.c head/sys/netinet/toecore.h head/sys/sys/ktls.h Modified: head/sys/kern/kern_sendfile.c ============================================================================== --- head/sys/kern/kern_sendfile.c Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/kern/kern_sendfile.c Tue Oct 8 21:34:06 2019 (r353328) @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vnode.h> #include <net/vnet.h> +#include <netinet/tcp.h> #include <security/audit/audit.h> #include <security/mac/mac_framework.h> @@ -295,7 +296,7 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, i mb_free_notready(sfio->m, sfio->npages); #ifdef KERN_TLS - } else if (sfio->tls != NULL && sfio->tls->sw_encrypt != NULL) { + } else if (sfio->tls != NULL && sfio->tls->mode == TCP_TLS_MODE_SW) { /* * I/O operation is complete, but we still need to * encrypt. We cannot do this in the interrupt thread @@ -1028,7 +1029,7 @@ prepend_header: */ free(sfio, M_TEMP); #ifdef KERN_TLS - if (tls != NULL && tls->sw_encrypt != NULL) { + if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { error = (*so->so_proto->pr_usrreqs->pru_send) (so, PRUS_NOTREADY, m, NULL, NULL, td); soref(so); Modified: head/sys/kern/uipc_ktls.c ============================================================================== --- head/sys/kern/uipc_ktls.c Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/kern/uipc_ktls.c Tue Oct 8 21:34:06 2019 (r353328) @@ -63,6 +63,9 @@ __FBSDID("$FreeBSD$"); #include <netinet/in_pcb.h> #endif #include <netinet/tcp_var.h> +#ifdef TCP_OFFLOAD +#include <netinet/tcp_offload.h> +#endif #include <opencrypto/xform.h> #include <vm/uma_dbg.h> #include <vm/vm.h> @@ -161,6 +164,10 @@ SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD, 0 "Software TLS session stats"); SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD, 0, "Hardware (ifnet) TLS session stats"); +#ifdef TCP_OFFLOAD +SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD, 0, + "TOE TLS session stats"); +#endif static counter_u64_t ktls_sw_cbc; SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, @@ -199,6 +206,18 @@ SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, &ktls_ifnet_permitted, 1, "Whether to permit hardware (ifnet) TLS sessions"); +#ifdef TCP_OFFLOAD +static counter_u64_t ktls_toe_cbc; +SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD, + &ktls_toe_cbc, + "Active number of TOE TLS sessions using AES-CBC"); + +static counter_u64_t ktls_toe_gcm; +SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD, + &ktls_toe_gcm, + "Active number of TOE TLS sessions using AES-GCM"); +#endif + static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS"); static void ktls_cleanup(struct ktls_session *tls); @@ -325,6 +344,10 @@ ktls_init(void *dummy __unused) ktls_ifnet_reset = counter_u64_alloc(M_WAITOK); ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK); ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK); +#ifdef TCP_OFFLOAD + ktls_toe_cbc = counter_u64_alloc(M_WAITOK); + ktls_toe_gcm = counter_u64_alloc(M_WAITOK); +#endif rm_init(&ktls_backends_lock, "ktls backends"); LIST_INIT(&ktls_backends); @@ -607,7 +630,8 @@ ktls_cleanup(struct ktls_session *tls) { counter_u64_add(ktls_offload_active, -1); - if (tls->free != NULL) { + switch (tls->mode) { + case TCP_TLS_MODE_SW: MPASS(tls->be != NULL); switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: @@ -618,7 +642,8 @@ ktls_cleanup(struct ktls_session *tls) break; } tls->free(tls); - } else if (tls->snd_tag != NULL) { + break; + case TCP_TLS_MODE_IFNET: switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_ifnet_cbc, -1); @@ -628,6 +653,19 @@ ktls_cleanup(struct ktls_session *tls) break; } m_snd_tag_rele(tls->snd_tag); + break; +#ifdef TCP_OFFLOAD + case TCP_TLS_MODE_TOE: + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_toe_cbc, -1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_toe_gcm, -1); + break; + } + break; +#endif } if (tls->params.auth_key != NULL) { explicit_bzero(tls->params.auth_key, tls->params.auth_key_len); @@ -646,6 +684,52 @@ ktls_cleanup(struct ktls_session *tls) } #if defined(INET) || defined(INET6) + +#ifdef TCP_OFFLOAD +static int +ktls_try_toe(struct socket *so, struct ktls_session *tls) +{ + struct inpcb *inp; + struct tcpcb *tp; + int error; + + inp = so->so_pcb; + INP_WLOCK(inp); + if (inp->inp_flags2 & INP_FREED) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + if (inp->inp_socket == NULL) { + INP_WUNLOCK(inp); + return (ECONNRESET); + } + tp = intotcpcb(inp); + if (tp->tod == NULL) { + INP_WUNLOCK(inp); + return (EOPNOTSUPP); + } + + error = tcp_offload_alloc_tls_session(tp, tls); + INP_WUNLOCK(inp); + if (error == 0) { + tls->mode = TCP_TLS_MODE_TOE; + switch (tls->params.cipher_algorithm) { + case CRYPTO_AES_CBC: + counter_u64_add(ktls_toe_cbc, 1); + break; + case CRYPTO_AES_NIST_GCM_16: + counter_u64_add(ktls_toe_gcm, 1); + break; + } + } + return (error); +} +#endif + /* * Common code used when first enabling ifnet TLS on a connection or * when allocating a new ifnet TLS session due to a routing change. @@ -744,6 +828,7 @@ ktls_try_ifnet(struct socket *so, struct ktls_session error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst); if (error == 0) { + tls->mode = TCP_TLS_MODE_IFNET; tls->snd_tag = mst; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: @@ -787,6 +872,7 @@ ktls_try_sw(struct socket *so, struct ktls_session *tl rm_runlock(&ktls_backends_lock, &prio); if (be == NULL) return (EOPNOTSUPP); + tls->mode = TCP_TLS_MODE_SW; switch (tls->params.cipher_algorithm) { case CRYPTO_AES_CBC: counter_u64_add(ktls_sw_cbc, 1); @@ -834,9 +920,13 @@ ktls_enable_tx(struct socket *so, struct tls_enable *e if (error) return (error); - /* Prefer ifnet TLS over software TLS. */ - error = ktls_try_ifnet(so, tls, false); + /* Prefer TOE -> ifnet TLS -> software TLS. */ +#ifdef TCP_OFFLOAD + error = ktls_try_toe(so, tls); if (error) +#endif + error = ktls_try_ifnet(so, tls, false); + if (error) error = ktls_try_sw(so, tls); if (error) { @@ -852,7 +942,7 @@ ktls_enable_tx(struct socket *so, struct tls_enable *e SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls; - if (tls->sw_encrypt == NULL) + if (tls->mode != TCP_TLS_MODE_SW) so->so_snd.sb_flags |= SB_TLS_IFNET; SOCKBUF_UNLOCK(&so->so_snd); sbunlock(&so->so_snd); @@ -875,10 +965,8 @@ ktls_get_tx_mode(struct socket *so) tls = so->so_snd.sb_tls_info; if (tls == NULL) mode = TCP_TLS_MODE_NONE; - else if (tls->sw_encrypt != NULL) - mode = TCP_TLS_MODE_SW; else - mode = TCP_TLS_MODE_IFNET; + mode = tls->mode; SOCKBUF_UNLOCK(&so->so_snd); return (mode); } @@ -893,7 +981,13 @@ ktls_set_tx_mode(struct socket *so, int mode) struct inpcb *inp; int error; - MPASS(mode == TCP_TLS_MODE_SW || mode == TCP_TLS_MODE_IFNET); + switch (mode) { + case TCP_TLS_MODE_SW: + case TCP_TLS_MODE_IFNET: + break; + default: + return (EINVAL); + } inp = so->so_pcb; INP_WLOCK_ASSERT(inp); @@ -904,8 +998,7 @@ ktls_set_tx_mode(struct socket *so, int mode) return (0); } - if ((tls->sw_encrypt != NULL && mode == TCP_TLS_MODE_SW) || - (tls->sw_encrypt == NULL && mode == TCP_TLS_MODE_IFNET)) { + if (tls->mode == mode) { SOCKBUF_UNLOCK(&so->so_snd); return (0); } @@ -952,7 +1045,7 @@ ktls_set_tx_mode(struct socket *so, int mode) SOCKBUF_LOCK(&so->so_snd); so->so_snd.sb_tls_info = tls_new; - if (tls_new->sw_encrypt == NULL) + if (tls_new->mode != TCP_TLS_MODE_SW) so->so_snd.sb_flags |= SB_TLS_IFNET; SOCKBUF_UNLOCK(&so->so_snd); sbunlock(&so->so_snd); @@ -1238,7 +1331,7 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, * When using ifnet TLS, unencrypted TLS records are * sent down the stack to the NIC. */ - if (tls->sw_encrypt != NULL) { + if (tls->mode == TCP_TLS_MODE_SW) { m->m_flags |= M_NOTREADY; pgs->nrdy = pgs->npgs; *enq_cnt += pgs->npgs; @@ -1278,7 +1371,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int pa pgs = m->m_ext.ext_pgs; - KASSERT(pgs->tls->sw_encrypt != NULL, ("ifnet TLS mbuf")); + KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf")); pgs->enc_cnt = page_count; pgs->mbuf = m; Modified: head/sys/kern/uipc_socket.c ============================================================================== --- head/sys/kern/uipc_socket.c Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/kern/uipc_socket.c Tue Oct 8 21:34:06 2019 (r353328) @@ -1491,7 +1491,7 @@ sosend_generic(struct socket *so, struct sockaddr *add tls_pruflag = 0; tls = ktls_hold(so->so_snd.sb_tls_info); if (tls != NULL) { - if (tls->sw_encrypt != NULL) + if (tls->mode == TCP_TLS_MODE_SW) tls_pruflag = PRUS_NOTREADY; if (control != NULL) { @@ -1659,7 +1659,7 @@ restart: } #ifdef KERN_TLS - if (tls != NULL && tls->sw_encrypt != NULL) { + if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) { /* * Note that error is intentionally * ignored. Modified: head/sys/netinet/tcp.h ============================================================================== --- head/sys/netinet/tcp.h Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/netinet/tcp.h Tue Oct 8 21:34:06 2019 (r353328) @@ -357,6 +357,7 @@ struct tcp_function_set { #define TCP_TLS_MODE_NONE 0 #define TCP_TLS_MODE_SW 1 #define TCP_TLS_MODE_IFNET 2 +#define TCP_TLS_MODE_TOE 3 /* * TCP Control message types Modified: head/sys/netinet/tcp_offload.c ============================================================================== --- head/sys/netinet/tcp_offload.c Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/netinet/tcp_offload.c Tue Oct 8 21:34:06 2019 (r353328) @@ -178,6 +178,17 @@ tcp_offload_tcp_info(struct tcpcb *tp, struct tcp_info tod->tod_tcp_info(tod, tp, ti); } +int +tcp_offload_alloc_tls_session(struct tcpcb *tp, struct ktls_session *tls) +{ + struct toedev *tod = tp->tod; + + KASSERT(tod != NULL, ("%s: tp->tod is NULL, tp %p", __func__, tp)); + INP_WLOCK_ASSERT(tp->t_inpcb); + + return (tod->tod_alloc_tls_session(tod, tp, tls)); +} + void tcp_offload_detach(struct tcpcb *tp) { Modified: head/sys/netinet/tcp_offload.h ============================================================================== --- head/sys/netinet/tcp_offload.h Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/netinet/tcp_offload.h Tue Oct 8 21:34:06 2019 (r353328) @@ -46,6 +46,7 @@ int tcp_offload_output(struct tcpcb *); void tcp_offload_rcvd(struct tcpcb *); void tcp_offload_ctloutput(struct tcpcb *, int, int); void tcp_offload_tcp_info(struct tcpcb *, struct tcp_info *); +int tcp_offload_alloc_tls_session(struct tcpcb *, struct ktls_session *); void tcp_offload_detach(struct tcpcb *); #endif Modified: head/sys/netinet/tcp_usrreq.c ============================================================================== --- head/sys/netinet/tcp_usrreq.c Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/netinet/tcp_usrreq.c Tue Oct 8 21:34:06 2019 (r353328) @@ -1936,8 +1936,6 @@ unlock_and_done: error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui)); if (error) return (error); - if (ui != TCP_TLS_MODE_SW && ui != TCP_TLS_MODE_IFNET) - return (EINVAL); INP_WLOCK_RECHECK(inp); error = ktls_set_tx_mode(so, ui); Modified: head/sys/netinet/toecore.c ============================================================================== --- head/sys/netinet/toecore.c Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/netinet/toecore.c Tue Oct 8 21:34:06 2019 (r353328) @@ -191,6 +191,14 @@ toedev_tcp_info(struct toedev *tod __unused, struct tc return; } +static int +toedev_alloc_tls_session(struct toedev *tod __unused, struct tcpcb *tp __unused, + struct ktls_session *tls __unused) +{ + + return (EINVAL); +} + /* * Inform one or more TOE devices about a listening socket. */ @@ -281,6 +289,7 @@ init_toedev(struct toedev *tod) tod->tod_offload_socket = toedev_offload_socket; tod->tod_ctloutput = toedev_ctloutput; tod->tod_tcp_info = toedev_tcp_info; + tod->tod_alloc_tls_session = toedev_alloc_tls_session; } /* Modified: head/sys/netinet/toecore.h ============================================================================== --- head/sys/netinet/toecore.h Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/netinet/toecore.h Tue Oct 8 21:34:06 2019 (r353328) @@ -41,6 +41,7 @@ struct tcpopt; struct tcphdr; struct in_conninfo; struct tcp_info; +struct ktls_session; struct toedev { TAILQ_ENTRY(toedev) link; /* glue for toedev_list */ @@ -108,6 +109,10 @@ struct toedev { /* Update software state */ void (*tod_tcp_info)(struct toedev *, struct tcpcb *, struct tcp_info *); + + /* Create a TLS session */ + int (*tod_alloc_tls_session)(struct toedev *, struct tcpcb *, + struct ktls_session *); }; typedef void (*tcp_offload_listen_start_fn)(void *, struct tcpcb *); Modified: head/sys/sys/ktls.h ============================================================================== --- head/sys/sys/ktls.h Tue Oct 8 21:14:11 2019 (r353327) +++ head/sys/sys/ktls.h Tue Oct 8 21:34:06 2019 (r353328) @@ -156,6 +156,7 @@ struct ktls_session { struct tls_session_params params; u_int wq_index; volatile u_int refcount; + int mode; struct task reset_tag_task; struct inpcb *inp;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201910082134.x98LY7YB088492>