Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 11 May 2016 08:22:58 -0700
From:      Adrian Chadd <adrian.chadd@gmail.com>
To:        Zbigniew Bodek <zbb@freebsd.org>
Cc:        "src-committers@freebsd.org" <src-committers@freebsd.org>,  "svn-src-all@freebsd.org" <svn-src-all@freebsd.org>,  "svn-src-head@freebsd.org" <svn-src-head@freebsd.org>
Subject:   Re: svn commit: r299444 - head/sys/dev/vnic
Message-ID:  <CAJ-Vmom1kusWGNz_5hSpR7zRYyHO3-z_E7L0ByoruMVAXv3MBw@mail.gmail.com>
In-Reply-To: <201605111322.u4BDMEsp070569@repo.freebsd.org>
References:  <201605111322.u4BDMEsp070569@repo.freebsd.org>

next in thread | previous in thread | raw e-mail | index | archive | help
ooo! do you have any plans to integrate it into the kernel RSS options?


-a


On 11 May 2016 at 06:22, Zbigniew Bodek <zbb@freebsd.org> wrote:
> Author: zbb
> Date: Wed May 11 13:22:13 2016
> New Revision: 299444
> URL: https://svnweb.freebsd.org/changeset/base/299444
>
> Log:
>   Add HW RSS support to VNIC driver
>
>   Based on v1.0 driver provided by Cavium under BSD license.
>   Support in-hardware RSS to distribute IP, UDP and TCP traffic
>   among available RX Queues and hence multiple CPUs.
>
>   Reviewed by:  wma
>   Obtained from:        Semihalf
>   Sponsored by: Cavium
>   Differential Revision: https://reviews.freebsd.org/D6230
>
> Modified:
>   head/sys/dev/vnic/nic.h
>   head/sys/dev/vnic/nic_main.c
>   head/sys/dev/vnic/nicvf_main.c
>   head/sys/dev/vnic/nicvf_queues.c
>
> Modified: head/sys/dev/vnic/nic.h
> ==============================================================================
> --- head/sys/dev/vnic/nic.h     Wed May 11 13:20:29 2016        (r299443)
> +++ head/sys/dev/vnic/nic.h     Wed May 11 13:22:13 2016        (r299444)
> @@ -176,6 +176,24 @@ struct msix_entry {
>  #define        NIC_MAX_RSS_IDR_TBL_SIZE        (1 << NIC_MAX_RSS_HASH_BITS)
>  #define        RSS_HASH_KEY_SIZE               5 /* 320 bit key */
>
> +struct nicvf_rss_info {
> +       boolean_t enable;
> +#define        RSS_L2_EXTENDED_HASH_ENA        (1UL << 0)
> +#define        RSS_IP_HASH_ENA                 (1UL << 1)
> +#define        RSS_TCP_HASH_ENA                (1UL << 2)
> +#define        RSS_TCP_SYN_DIS                 (1UL << 3)
> +#define        RSS_UDP_HASH_ENA                (1UL << 4)
> +#define        RSS_L4_EXTENDED_HASH_ENA        (1UL << 5)
> +#define        RSS_ROCE_ENA                    (1UL << 6)
> +#define        RSS_L3_BI_DIRECTION_ENA         (1UL << 7)
> +#define        RSS_L4_BI_DIRECTION_ENA         (1UL << 8)
> +       uint64_t cfg;
> +       uint8_t  hash_bits;
> +       uint16_t rss_size;
> +       uint8_t  ind_tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
> +       uint64_t key[RSS_HASH_KEY_SIZE];
> +};
> +
>  enum rx_stats_reg_offset {
>         RX_OCTS = 0x0,
>         RX_UCAST = 0x1,
> @@ -285,6 +303,7 @@ struct nicvf {
>         boolean_t               tns_mode:1;
>         boolean_t               sqs_mode:1;
>         bool                    loopback_supported:1;
> +       struct nicvf_rss_info   rss_info;
>         uint16_t                mtu;
>         struct queue_set        *qs;
>         uint8_t                 rx_queues;
>
> Modified: head/sys/dev/vnic/nic_main.c
> ==============================================================================
> --- head/sys/dev/vnic/nic_main.c        Wed May 11 13:20:29 2016        (r299443)
> +++ head/sys/dev/vnic/nic_main.c        Wed May 11 13:22:13 2016        (r299444)
> @@ -103,6 +103,7 @@ struct nicpf {
>         uint8_t                 duplex[MAX_LMAC];
>         uint32_t                speed[MAX_LMAC];
>         uint16_t                cpi_base[MAX_NUM_VFS_SUPPORTED];
> +       uint16_t                rssi_base[MAX_NUM_VFS_SUPPORTED];
>         uint16_t                rss_ind_tbl_size;
>
>         /* MSI-X */
> @@ -744,6 +745,58 @@ nic_config_cpi(struct nicpf *nic, struct
>                         rssi = ((cpi - cpi_base) & 0x38) >> 3;
>         }
>         nic->cpi_base[cfg->vf_id] = cpi_base;
> +       nic->rssi_base[cfg->vf_id] = rssi_base;
> +}
> +
> +/* Responsds to VF with its RSS indirection table size */
> +static void
> +nic_send_rss_size(struct nicpf *nic, int vf)
> +{
> +       union nic_mbx mbx = {};
> +       uint64_t  *msg;
> +
> +       msg = (uint64_t *)&mbx;
> +
> +       mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
> +       mbx.rss_size.ind_tbl_size = nic->rss_ind_tbl_size;
> +       nic_send_msg_to_vf(nic, vf, &mbx);
> +}
> +
> +/*
> + * Receive side scaling configuration
> + * configure:
> + * - RSS index
> + * - indir table i.e hash::RQ mapping
> + * - no of hash bits to consider
> + */
> +static void
> +nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg)
> +{
> +       uint8_t qset, idx;
> +       uint64_t cpi_cfg, cpi_base, rssi_base, rssi;
> +       uint64_t idx_addr;
> +
> +       idx = 0;
> +       rssi_base = nic->rssi_base[cfg->vf_id] + cfg->tbl_offset;
> +
> +       rssi = rssi_base;
> +       qset = cfg->vf_id;
> +
> +       for (; rssi < (rssi_base + cfg->tbl_len); rssi++) {
> +               nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3),
> +                   (qset << 3) | (cfg->ind_tbl[idx] & 0x7));
> +               idx++;
> +       }
> +
> +       cpi_base = nic->cpi_base[cfg->vf_id];
> +       if (pass1_silicon(nic->dev))
> +               idx_addr = NIC_PF_CPI_0_2047_CFG;
> +       else
> +               idx_addr = NIC_PF_MPI_0_2047_CFG;
> +       cpi_cfg = nic_reg_read(nic, idx_addr | (cpi_base << 3));
> +       cpi_cfg &= ~(0xFUL << 20);
> +       cpi_cfg |= (cfg->hash_bits << 20);
> +       nic_reg_write(nic, idx_addr | (cpi_base << 3), cpi_cfg);
>  }
>
>  /*
> @@ -896,6 +949,13 @@ nic_handle_mbx_intr(struct nicpf *nic, i
>         case NIC_MBOX_MSG_CPI_CFG:
>                 nic_config_cpi(nic, &mbx.cpi_cfg);
>                 break;
> +       case NIC_MBOX_MSG_RSS_SIZE:
> +               nic_send_rss_size(nic, vf);
> +               goto unlock;
> +       case NIC_MBOX_MSG_RSS_CFG:
> +       case NIC_MBOX_MSG_RSS_CFG_CONT: /* fall through */
> +               nic_config_rss(nic, &mbx.rss_cfg);
> +               break;
>         case NIC_MBOX_MSG_CFG_DONE:
>                 /* Last message of VF config msg sequence */
>                 nic->vf_info[vf].vf_enabled = TRUE;
>
> Modified: head/sys/dev/vnic/nicvf_main.c
> ==============================================================================
> --- head/sys/dev/vnic/nicvf_main.c      Wed May 11 13:20:29 2016        (r299443)
> +++ head/sys/dev/vnic/nicvf_main.c      Wed May 11 13:22:13 2016        (r299444)
> @@ -140,6 +140,7 @@ static int nicvf_allocate_net_interrupts
>  static void nicvf_release_all_interrupts(struct nicvf *);
>  static int nicvf_hw_set_mac_addr(struct nicvf *, uint8_t *);
>  static void nicvf_config_cpi(struct nicvf *);
> +static int nicvf_rss_init(struct nicvf *);
>  static int nicvf_init_resources(struct nicvf *);
>
>  static int nicvf_setup_ifnet(struct nicvf *);
> @@ -245,6 +246,9 @@ nicvf_attach(device_t dev)
>         nic->cpi_alg = CPI_ALG_NONE;
>         NICVF_CORE_LOCK(nic);
>         nicvf_config_cpi(nic);
> +       /* Configure receive side scaling */
> +       if (nic->qs->rq_cnt > 1)
> +               nicvf_rss_init(nic);
>         NICVF_CORE_UNLOCK(nic);
>
>         err = nicvf_setup_ifnet(nic);
> @@ -940,6 +944,10 @@ nicvf_handle_mbx_intr(struct nicvf *nic)
>         case NIC_MBOX_MSG_NACK:
>                 nic->pf_nacked = TRUE;
>                 break;
> +       case NIC_MBOX_MSG_RSS_SIZE:
> +               nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
> +               nic->pf_acked = TRUE;
> +               break;
>         case NIC_MBOX_MSG_BGX_STATS:
>                 nicvf_read_bgx_stats(nic, &mbx.bgx_stats);
>                 nic->pf_acked = TRUE;
> @@ -990,6 +998,100 @@ nicvf_config_cpi(struct nicvf *nic)
>         nicvf_send_msg_to_pf(nic, &mbx);
>  }
>
> +static void
> +nicvf_get_rss_size(struct nicvf *nic)
> +{
> +       union nic_mbx mbx = {};
> +
> +       mbx.rss_size.msg = NIC_MBOX_MSG_RSS_SIZE;
> +       mbx.rss_size.vf_id = nic->vf_id;
> +       nicvf_send_msg_to_pf(nic, &mbx);
> +}
> +
> +static void
> +nicvf_config_rss(struct nicvf *nic)
> +{
> +       union nic_mbx mbx = {};
> +       struct nicvf_rss_info *rss;
> +       int ind_tbl_len;
> +       int i, nextq;
> +
> +       rss = &nic->rss_info;
> +       ind_tbl_len = rss->rss_size;
> +       nextq = 0;
> +
> +       mbx.rss_cfg.vf_id = nic->vf_id;
> +       mbx.rss_cfg.hash_bits = rss->hash_bits;
> +       while (ind_tbl_len != 0) {
> +               mbx.rss_cfg.tbl_offset = nextq;
> +               mbx.rss_cfg.tbl_len = MIN(ind_tbl_len,
> +                   RSS_IND_TBL_LEN_PER_MBX_MSG);
> +               mbx.rss_cfg.msg = mbx.rss_cfg.tbl_offset ?
> +                   NIC_MBOX_MSG_RSS_CFG_CONT : NIC_MBOX_MSG_RSS_CFG;
> +
> +               for (i = 0; i < mbx.rss_cfg.tbl_len; i++)
> +                       mbx.rss_cfg.ind_tbl[i] = rss->ind_tbl[nextq++];
> +
> +               nicvf_send_msg_to_pf(nic, &mbx);
> +
> +               ind_tbl_len -= mbx.rss_cfg.tbl_len;
> +       }
> +}
> +
> +static void
> +nicvf_set_rss_key(struct nicvf *nic)
> +{
> +       struct nicvf_rss_info *rss;
> +       uint64_t key_addr;
> +       int idx;
> +
> +       rss = &nic->rss_info;
> +       key_addr = NIC_VNIC_RSS_KEY_0_4;
> +
> +       for (idx = 0; idx < RSS_HASH_KEY_SIZE; idx++) {
> +               nicvf_reg_write(nic, key_addr, rss->key[idx]);
> +               key_addr += sizeof(uint64_t);
> +       }
> +}
> +
> +static int
> +nicvf_rss_init(struct nicvf *nic)
> +{
> +       struct nicvf_rss_info *rss;
> +       int idx;
> +
> +       nicvf_get_rss_size(nic);
> +
> +       rss = &nic->rss_info;
> +       if (nic->cpi_alg != CPI_ALG_NONE) {
> +               rss->enable = FALSE;
> +               rss->hash_bits = 0;
> +               return (ENXIO);
> +       }
> +
> +       rss->enable = TRUE;
> +
> +       /* Using the HW reset value for now */
> +       rss->key[0] = 0xFEED0BADFEED0BADUL;
> +       rss->key[1] = 0xFEED0BADFEED0BADUL;
> +       rss->key[2] = 0xFEED0BADFEED0BADUL;
> +       rss->key[3] = 0xFEED0BADFEED0BADUL;
> +       rss->key[4] = 0xFEED0BADFEED0BADUL;
> +
> +       nicvf_set_rss_key(nic);
> +
> +       rss->cfg = RSS_IP_HASH_ENA | RSS_TCP_HASH_ENA | RSS_UDP_HASH_ENA;
> +       nicvf_reg_write(nic, NIC_VNIC_RSS_CFG, rss->cfg);
> +
> +       rss->hash_bits = fls(rss->rss_size) - 1;
> +       for (idx = 0; idx < rss->rss_size; idx++)
> +               rss->ind_tbl[idx] = idx % nic->rx_queues;
> +
> +       nicvf_config_rss(nic);
> +
> +       return (0);
> +}
> +
>  static int
>  nicvf_init_resources(struct nicvf *nic)
>  {
>
> Modified: head/sys/dev/vnic/nicvf_queues.c
> ==============================================================================
> --- head/sys/dev/vnic/nicvf_queues.c    Wed May 11 13:20:29 2016        (r299443)
> +++ head/sys/dev/vnic/nicvf_queues.c    Wed May 11 13:22:13 2016        (r299444)
> @@ -1611,8 +1611,7 @@ nicvf_set_qset_resources(struct nicvf *n
>
>         /* Set count of each queue */
>         qs->rbdr_cnt = RBDR_CNT;
> -       /* With no RSS we stay with single RQ */
> -       qs->rq_cnt = 1;
> +       qs->rq_cnt = RCV_QUEUE_CNT;
>
>         qs->sq_cnt = SND_QUEUE_CNT;
>         qs->cq_cnt = CMP_QUEUE_CNT;
>



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?CAJ-Vmom1kusWGNz_5hSpR7zRYyHO3-z_E7L0ByoruMVAXv3MBw>