Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 18 Oct 2015 22:02:58 +0000 (UTC)
From:      Zbigniew Bodek <zbb@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r289551 - in head/sys: conf dev/vnic
Message-ID:  <201510182202.t9IM2wL0025859@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: zbb
Date: Sun Oct 18 22:02:58 2015
New Revision: 289551
URL: https://svnweb.freebsd.org/changeset/base/289551

Log:
  Introduce initial support for Cavium's ThunderX networking interface
  
  - The driver consists of three main componens: PF, VF, BGX
  - Requires appropriate entries in DTS and MDIO driver
  - Supports only FDT configuration
  - Multiple Tx queues and single Rx queue supported
  - No RSS, HW checksum and TSO support
  - No more than 8 queues per-IF (only one Queue Set per IF)
  - HW statistics enabled
  - Works in all available MAC modes (1,10,20,40G)
  - Style converted to BSD according to style(9)
  - The code brings lmac_if interface used by the BGX driver to
    update its logical MACs state.
  
  Obtained from: Semihalf
  Sponsored by:  The FreeBSD Foundation

Added:
  head/sys/dev/vnic/lmac_if.m   (contents, props changed)
  head/sys/dev/vnic/thunder_bgx_fdt.c   (contents, props changed)
  head/sys/dev/vnic/thunder_bgx_var.h   (contents, props changed)
Modified:
  head/sys/conf/files.arm64
  head/sys/dev/vnic/nic.h
  head/sys/dev/vnic/nic_main.c
  head/sys/dev/vnic/nic_reg.h
  head/sys/dev/vnic/nicvf_main.c
  head/sys/dev/vnic/nicvf_queues.c
  head/sys/dev/vnic/nicvf_queues.h
  head/sys/dev/vnic/q_struct.h
  head/sys/dev/vnic/thunder_bgx.c
  head/sys/dev/vnic/thunder_bgx.h

Modified: head/sys/conf/files.arm64
==============================================================================
--- head/sys/conf/files.arm64	Sun Oct 18 21:39:15 2015	(r289550)
+++ head/sys/conf/files.arm64	Sun Oct 18 22:02:58 2015	(r289551)
@@ -68,6 +68,12 @@ dev/psci/psci_arm64.S		optional	psci
 dev/uart/uart_cpu_fdt.c		optional	uart fdt
 dev/uart/uart_dev_pl011.c	optional	uart pl011
 dev/usb/controller/dwc_otg_hisi.c optional	dwcotg soc_hisi_hi6220
+dev/vnic/nic_main.c		optional	vnic pci
+dev/vnic/nicvf_main.c		optional	vnic pci pci_iov
+dev/vnic/nicvf_queues.c		optional	vnic pci pci_iov
+dev/vnic/thunder_bgx_fdt.c	optional	vnic fdt
+dev/vnic/thunder_bgx.c		optional	vnic pci
+dev/vnic/lmac_if.m		optional	vnic
 kern/kern_clocksource.c		standard
 kern/subr_dummy_vdso_tc.c	standard
 libkern/bcmp.c			standard

Added: head/sys/dev/vnic/lmac_if.m
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/sys/dev/vnic/lmac_if.m	Sun Oct 18 22:02:58 2015	(r289551)
@@ -0,0 +1,102 @@
+#-
+# Copyright (c) 2015 The FreeBSD Foundation
+# All rights reserved.
+#
+# This software was developed by Semihalf under
+# the sponsorship of the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+
+# LMAC (BGX controller) interface description
+#
+
+INTERFACE lmac;
+
+CODE {
+	static int null_lmac_media_status(device_t dev, int lmacid, int *link,
+	    int *duplex, int *speed)
+	{
+		return (ENXIO);
+	}
+
+	static int null_lmac_media_change(device_t dev, int lmacid, int link,
+	    int duplex, int speed)
+	{
+		return (ENXIO);
+	}
+
+	static int null_lmac_phy_connect(device_t dev, int lmacid, int phy)
+	{
+		return (ENXIO);
+	}
+
+	static int null_lmac_phy_disconnect(device_t dev, int lmacid, int phy)
+	{
+		return (ENXIO);
+	}
+};
+
+# Get link status
+#
+# 0 : Success
+#
+METHOD int media_status {
+	device_t		dev;
+	int			lmacid;
+	int *			link;
+	int *			duplex;
+	int *			speed;
+} DEFAULT null_lmac_media_status;
+
+# Change link status
+#
+# 0 : Success
+#
+METHOD int media_change {
+	device_t		dev;
+	int			lmacid;
+	int			link;
+	int			duplex;
+	int			speed;
+} DEFAULT null_lmac_media_change;
+
+# Connect PHY
+#
+# 0 : Success
+#
+METHOD int phy_connect {
+	device_t		dev;
+	int			lmacid;
+	int			phy;
+} DEFAULT null_lmac_phy_connect;
+
+# Disconnect PHY
+#
+# 0 : Success
+#
+METHOD int phy_disconnect {
+	device_t		dev;
+	int			lmacid;
+	int			phy;
+} DEFAULT null_lmac_phy_disconnect;

Modified: head/sys/dev/vnic/nic.h
==============================================================================
--- head/sys/dev/vnic/nic.h	Sun Oct 18 21:39:15 2015	(r289550)
+++ head/sys/dev/vnic/nic.h	Sun Oct 18 22:02:58 2015	(r289551)
@@ -30,11 +30,8 @@
 #ifndef NIC_H
 #define	NIC_H
 
-#include <linux/netdevice.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include "thunder_bgx.h"
-
+/* PCI vendor ID */
+#define PCI_VENDOR_ID_CAVIUM			0x177D
 /* PCI device IDs */
 #define	PCI_DEVICE_ID_THUNDER_NIC_PF		0xA01E
 #define	PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF	0x0011
@@ -53,12 +50,15 @@
 #define	NIC_TNS_MODE			1
 
 /* NIC priv flags */
-#define	NIC_SRIOV_ENABLED		BIT(0)
-#define	NIC_TNS_ENABLED			BIT(1)
+#define	NIC_SRIOV_ENABLED		(1 << 0)
+#define	NIC_TNS_ENABLED			(1 << 1)
 
+/* ARM64TODO */
+#if 0
 /* VNIC HW optimiation features */
 #define VNIC_RSS_SUPPORT
 #define VNIC_MULTI_QSET_SUPPORT
+#endif
 
 /* Min/Max packet size */
 #define	NIC_HW_MIN_FRS			64
@@ -67,7 +67,8 @@
 /* Max pkinds */
 #define	NIC_MAX_PKIND			16
 
-/* Rx Channels */
+/*
+ * Rx Channels */
 /* Receive channel configuration in TNS bypass mode
  * Below is configuration in TNS bypass mode
  * BGX0-LMAC0-CHAN0 - VNIC CHAN0
@@ -83,7 +84,7 @@
 #define	NIC_CPI_COUNT			2048 /* No of channel parse indices */
 
 /* TNS bypass mode: 1-1 mapping between VNIC and BGX:LMAC */
-#define NIC_MAX_BGX			MAX_BGX_PER_CN88XX
+#define	NIC_MAX_BGX			MAX_BGX_PER_CN88XX
 #define	NIC_CPI_PER_BGX			(NIC_CPI_COUNT / NIC_MAX_BGX)
 #define	NIC_MAX_CPI_PER_LMAC		64 /* Max when CPI_ALG is IP diffserv */
 #define	NIC_RSSI_PER_BGX		(NIC_RSSI_COUNT / NIC_MAX_BGX)
@@ -122,27 +123,33 @@
 #define	NICVF_INTR_CQ_MASK		(0xFF << NICVF_INTR_CQ_SHIFT)
 #define	NICVF_INTR_SQ_MASK		(0xFF << NICVF_INTR_SQ_SHIFT)
 #define	NICVF_INTR_RBDR_MASK		(0x03 << NICVF_INTR_RBDR_SHIFT)
-#define	NICVF_INTR_PKT_DROP_MASK	BIT(NICVF_INTR_PKT_DROP_SHIFT)
-#define	NICVF_INTR_TCP_TIMER_MASK	BIT(NICVF_INTR_TCP_TIMER_SHIFT)
-#define	NICVF_INTR_MBOX_MASK		BIT(NICVF_INTR_MBOX_SHIFT)
-#define	NICVF_INTR_QS_ERR_MASK		BIT(NICVF_INTR_QS_ERR_SHIFT)
+#define	NICVF_INTR_PKT_DROP_MASK	(1 << NICVF_INTR_PKT_DROP_SHIFT)
+#define	NICVF_INTR_TCP_TIMER_MASK	(1 << NICVF_INTR_TCP_TIMER_SHIFT)
+#define	NICVF_INTR_MBOX_MASK		(1 << NICVF_INTR_MBOX_SHIFT)
+#define	NICVF_INTR_QS_ERR_MASK		(1 << NICVF_INTR_QS_ERR_SHIFT)
 
 /* MSI-X interrupts */
 #define	NIC_PF_MSIX_VECTORS		10
 #define	NIC_VF_MSIX_VECTORS		20
 
-#define NIC_PF_INTR_ID_ECC0_SBE		0
-#define NIC_PF_INTR_ID_ECC0_DBE		1
-#define NIC_PF_INTR_ID_ECC1_SBE		2
-#define NIC_PF_INTR_ID_ECC1_DBE		3
-#define NIC_PF_INTR_ID_ECC2_SBE		4
-#define NIC_PF_INTR_ID_ECC2_DBE		5
-#define NIC_PF_INTR_ID_ECC3_SBE		6
-#define NIC_PF_INTR_ID_ECC3_DBE		7
-#define NIC_PF_INTR_ID_MBOX0		8
-#define NIC_PF_INTR_ID_MBOX1		9
+#define	NIC_PF_INTR_ID_ECC0_SBE		0
+#define	NIC_PF_INTR_ID_ECC0_DBE		1
+#define	NIC_PF_INTR_ID_ECC1_SBE		2
+#define	NIC_PF_INTR_ID_ECC1_DBE		3
+#define	NIC_PF_INTR_ID_ECC2_SBE		4
+#define	NIC_PF_INTR_ID_ECC2_DBE		5
+#define	NIC_PF_INTR_ID_ECC3_SBE		6
+#define	NIC_PF_INTR_ID_ECC3_DBE		7
+#define	NIC_PF_INTR_ID_MBOX0		8
+#define	NIC_PF_INTR_ID_MBOX1		9
+
+struct msix_entry {
+	struct resource *	irq_res;
+	void *			handle;
+};
 
-/* Global timer for CQ timer thresh interrupts
+/*
+ * Global timer for CQ timer thresh interrupts
  * Calculated for SCLK of 700Mhz
  * value written should be a 1/16th of what is expected
  *
@@ -151,7 +158,8 @@
  */
 #define NICPF_CLK_PER_INT_TICK		2
 
-/* Time to wait before we decide that a SQ is stuck.
+/*
+ * Time to wait before we decide that a SQ is stuck.
  *
  * Since both pkt rx and tx notifications are done with same CQ,
  * when packets are being received at very high rate (eg: L2 forwarding)
@@ -160,36 +168,10 @@
  */
 #define	NICVF_TX_TIMEOUT		(50 * HZ)
 
-struct nicvf_cq_poll {
-	struct  nicvf *nicvf;
-	u8	cq_idx;		/* Completion queue index */
-	struct	napi_struct napi;
-};
-
 #define	NIC_RSSI_COUNT			4096 /* Total no of RSS indices */
-#define NIC_MAX_RSS_HASH_BITS		8
-#define NIC_MAX_RSS_IDR_TBL_SIZE	(1 << NIC_MAX_RSS_HASH_BITS)
-#define RSS_HASH_KEY_SIZE		5 /* 320 bit key */
-
-#ifdef VNIC_RSS_SUPPORT
-struct nicvf_rss_info {
-	bool enable;
-#define	RSS_L2_EXTENDED_HASH_ENA	BIT(0)
-#define	RSS_IP_HASH_ENA			BIT(1)
-#define	RSS_TCP_HASH_ENA		BIT(2)
-#define	RSS_TCP_SYN_DIS			BIT(3)
-#define	RSS_UDP_HASH_ENA		BIT(4)
-#define RSS_L4_EXTENDED_HASH_ENA	BIT(5)
-#define	RSS_ROCE_ENA			BIT(6)
-#define	RSS_L3_BI_DIRECTION_ENA		BIT(7)
-#define	RSS_L4_BI_DIRECTION_ENA		BIT(8)
-	u64 cfg;
-	u8  hash_bits;
-	u16 rss_size;
-	u8  ind_tbl[NIC_MAX_RSS_IDR_TBL_SIZE];
-	u64 key[RSS_HASH_KEY_SIZE];
-} ____cacheline_aligned_in_smp;
-#endif
+#define	NIC_MAX_RSS_HASH_BITS		8
+#define	NIC_MAX_RSS_IDR_TBL_SIZE	(1 << NIC_MAX_RSS_HASH_BITS)
+#define	RSS_HASH_KEY_SIZE		5 /* 320 bit key */
 
 enum rx_stats_reg_offset {
 	RX_OCTS = 0x0,
@@ -219,132 +201,124 @@ enum tx_stats_reg_offset {
 };
 
 struct nicvf_hw_stats {
-	u64 rx_bytes;
-	u64 rx_ucast_frames;
-	u64 rx_bcast_frames;
-	u64 rx_mcast_frames;
-	u64 rx_fcs_errors;
-	u64 rx_l2_errors;
-	u64 rx_drop_red;
-	u64 rx_drop_red_bytes;
-	u64 rx_drop_overrun;
-	u64 rx_drop_overrun_bytes;
-	u64 rx_drop_bcast;
-	u64 rx_drop_mcast;
-	u64 rx_drop_l3_bcast;
-	u64 rx_drop_l3_mcast;
-	u64 rx_bgx_truncated_pkts;
-	u64 rx_jabber_errs;
-	u64 rx_fcs_errs;
-	u64 rx_bgx_errs;
-	u64 rx_prel2_errs;
-	u64 rx_l2_hdr_malformed;
-	u64 rx_oversize;
-	u64 rx_undersize;
-	u64 rx_l2_len_mismatch;
-	u64 rx_l2_pclp;
-	u64 rx_ip_ver_errs;
-	u64 rx_ip_csum_errs;
-	u64 rx_ip_hdr_malformed;
-	u64 rx_ip_payload_malformed;
-	u64 rx_ip_ttl_errs;
-	u64 rx_l3_pclp;
-	u64 rx_l4_malformed;
-	u64 rx_l4_csum_errs;
-	u64 rx_udp_len_errs;
-	u64 rx_l4_port_errs;
-	u64 rx_tcp_flag_errs;
-	u64 rx_tcp_offset_errs;
-	u64 rx_l4_pclp;
-	u64 rx_truncated_pkts;
-
-	u64 tx_bytes_ok;
-	u64 tx_ucast_frames_ok;
-	u64 tx_bcast_frames_ok;
-	u64 tx_mcast_frames_ok;
-	u64 tx_drops;
+	uint64_t rx_bytes;
+	uint64_t rx_ucast_frames;
+	uint64_t rx_bcast_frames;
+	uint64_t rx_mcast_frames;
+	uint64_t rx_fcs_errors;
+	uint64_t rx_l2_errors;
+	uint64_t rx_drop_red;
+	uint64_t rx_drop_red_bytes;
+	uint64_t rx_drop_overrun;
+	uint64_t rx_drop_overrun_bytes;
+	uint64_t rx_drop_bcast;
+	uint64_t rx_drop_mcast;
+	uint64_t rx_drop_l3_bcast;
+	uint64_t rx_drop_l3_mcast;
+	uint64_t rx_bgx_truncated_pkts;
+	uint64_t rx_jabber_errs;
+	uint64_t rx_fcs_errs;
+	uint64_t rx_bgx_errs;
+	uint64_t rx_prel2_errs;
+	uint64_t rx_l2_hdr_malformed;
+	uint64_t rx_oversize;
+	uint64_t rx_undersize;
+	uint64_t rx_l2_len_mismatch;
+	uint64_t rx_l2_pclp;
+	uint64_t rx_ip_ver_errs;
+	uint64_t rx_ip_csum_errs;
+	uint64_t rx_ip_hdr_malformed;
+	uint64_t rx_ip_payload_malformed;
+	uint64_t rx_ip_ttl_errs;
+	uint64_t rx_l3_pclp;
+	uint64_t rx_l4_malformed;
+	uint64_t rx_l4_csum_errs;
+	uint64_t rx_udp_len_errs;
+	uint64_t rx_l4_port_errs;
+	uint64_t rx_tcp_flag_errs;
+	uint64_t rx_tcp_offset_errs;
+	uint64_t rx_l4_pclp;
+	uint64_t rx_truncated_pkts;
+
+	uint64_t tx_bytes_ok;
+	uint64_t tx_ucast_frames_ok;
+	uint64_t tx_bcast_frames_ok;
+	uint64_t tx_mcast_frames_ok;
+	uint64_t tx_drops;
 };
 
 struct nicvf_drv_stats {
 	/* Rx */
-	u64 rx_frames_ok;
-	u64 rx_frames_64;
-	u64 rx_frames_127;
-	u64 rx_frames_255;
-	u64 rx_frames_511;
-	u64 rx_frames_1023;
-	u64 rx_frames_1518;
-	u64 rx_frames_jumbo;
-	u64 rx_drops;
+	uint64_t rx_frames_ok;
+	uint64_t rx_frames_64;
+	uint64_t rx_frames_127;
+	uint64_t rx_frames_255;
+	uint64_t rx_frames_511;
+	uint64_t rx_frames_1023;
+	uint64_t rx_frames_1518;
+	uint64_t rx_frames_jumbo;
+	uint64_t rx_drops;
 
 	/* Tx */
-	u64 tx_frames_ok;
-	u64 tx_drops;
-	u64 tx_tso;
-	u64 txq_stop;
-	u64 txq_wake;
+	uint64_t tx_frames_ok;
+	uint64_t tx_drops;
+	uint64_t tx_tso;
+	uint64_t txq_stop;
+	uint64_t txq_wake;
 };
 
 struct nicvf {
 	struct nicvf		*pnicvf;
-	struct net_device	*netdev;
-	struct pci_dev		*pdev;
-	u8			vf_id;
-	u8			node;
-	bool			tns_mode:1;
-	bool                    sqs_mode:1;
+	device_t		dev;
+
+	struct ifnet *		ifp;
+	struct sx		core_sx;
+	struct ifmedia		if_media;
+	uint32_t		if_flags;
+
+	uint8_t			hwaddr[ETHER_ADDR_LEN];
+	uint8_t			vf_id;
+	uint8_t			node;
+	boolean_t		tns_mode:1;
+	boolean_t		sqs_mode:1;
 	bool			loopback_supported:1;
-	u16			mtu;
+	uint16_t		mtu;
 	struct queue_set	*qs;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-#define	MAX_SQS_PER_VF_SINGLE_NODE		5
-#define	MAX_SQS_PER_VF				11
-	u8			sqs_id;
-	u8			sqs_count; /* Secondary Qset count */
-	struct nicvf		*snicvf[MAX_SQS_PER_VF];
-#endif
-	u8			rx_queues;
-	u8			tx_queues;
-	u8			max_queues;
-	void __iomem		*reg_base;
-	bool			link_up;
-	u8			duplex;
-	u32			speed;
-	struct page		*rb_page;
-	u32			rb_page_offset;
-	bool			rb_alloc_fail;
-	bool			rb_work_scheduled;
-	struct delayed_work	rbdr_work;
-	struct tasklet_struct	rbdr_task;
-	struct tasklet_struct	qs_err_task;
-	struct tasklet_struct	cq_task;
-	struct nicvf_cq_poll	*napi[8];
-#ifdef VNIC_RSS_SUPPORT
-	struct nicvf_rss_info	rss_info;
-#endif
-	u8			cpi_alg;
+	uint8_t			rx_queues;
+	uint8_t			tx_queues;
+	uint8_t			max_queues;
+	struct resource		*reg_base;
+	boolean_t		link_up;
+	uint8_t			duplex;
+	uint32_t		speed;
+	uint8_t			cpi_alg;
 	/* Interrupt coalescing settings */
-	u32			cq_coalesce_usecs;
+	uint32_t		cq_coalesce_usecs;
 
-	u32			msg_enable;
-	struct nicvf_hw_stats   hw_stats;
-	struct nicvf_drv_stats  drv_stats;
+	uint32_t		msg_enable;
+	struct nicvf_hw_stats	hw_stats;
+	struct nicvf_drv_stats	drv_stats;
 	struct bgx_stats	bgx_stats;
-	struct work_struct	reset_task;
+
+	/* Interface statistics */
+	struct callout		stats_callout;
+	struct mtx		stats_mtx;
 
 	/* MSI-X  */
-	bool			msix_enabled;
-	u8			num_vec;
+	boolean_t		msix_enabled;
+	uint8_t			num_vec;
 	struct msix_entry	msix_entries[NIC_VF_MSIX_VECTORS];
+	struct resource *	msix_table_res;
 	char			irq_name[NIC_VF_MSIX_VECTORS][20];
-	bool			irq_allocated[NIC_VF_MSIX_VECTORS];
+	boolean_t		irq_allocated[NIC_VF_MSIX_VECTORS];
 
 	/* VF <-> PF mailbox communication */
-	bool			pf_acked;
-	bool			pf_nacked;
-} ____cacheline_aligned_in_smp;
+	boolean_t		pf_acked;
+	boolean_t		pf_nacked;
+} __aligned(CACHE_LINE_SIZE);
 
-/* PF <--> VF Mailbox communication
+/*
+ * PF <--> VF Mailbox communication
  * Eight 64bit registers are shared between PF and VF.
  * Separate set for each VF.
  * Writing '1' into last register mbx7 means end of message.
@@ -381,123 +355,108 @@ struct nicvf {
 #define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
 
 struct nic_cfg_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    node_id;
-	bool  tns_mode:1;
-	bool  sqs_mode:1;
-	bool  loopback_supported:1;
-	u8    mac_addr[ETH_ALEN];
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		node_id;
+	boolean_t	tns_mode:1;
+	boolean_t	sqs_mode:1;
+	boolean_t	loopback_supported:1;
+	uint8_t	mac_addr[ETHER_ADDR_LEN];
 };
 
 /* Qset configuration */
 struct qs_cfg_msg {
-	u8    msg;
-	u8    num;
-	u8    sqs_count;
-	u64   cfg;
+	uint8_t		msg;
+	uint8_t		num;
+	uint8_t		sqs_count;
+	uint64_t	cfg;
 };
 
 /* Receive queue configuration */
 struct rq_cfg_msg {
-	u8    msg;
-	u8    qs_num;
-	u8    rq_num;
-	u64   cfg;
+	uint8_t		msg;
+	uint8_t		qs_num;
+	uint8_t		rq_num;
+	uint64_t	cfg;
 };
 
 /* Send queue configuration */
 struct sq_cfg_msg {
-	u8    msg;
-	u8    qs_num;
-	u8    sq_num;
-	bool  sqs_mode;
-	u64   cfg;
+	uint8_t		msg;
+	uint8_t		qs_num;
+	uint8_t		sq_num;
+	boolean_t	sqs_mode;
+	uint64_t	cfg;
 };
 
 /* Set VF's MAC address */
 struct set_mac_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    mac_addr[ETH_ALEN];
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		mac_addr[ETHER_ADDR_LEN];
 };
 
 /* Set Maximum frame size */
 struct set_frs_msg {
-	u8    msg;
-	u8    vf_id;
-	u16   max_frs;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint16_t	max_frs;
 };
 
 /* Set CPI algorithm type */
 struct cpi_cfg_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    rq_cnt;
-	u8    cpi_alg;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		rq_cnt;
+	uint8_t		cpi_alg;
 };
 
 /* Get RSS table size */
 struct rss_sz_msg {
-	u8    msg;
-	u8    vf_id;
-	u16   ind_tbl_size;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint16_t	ind_tbl_size;
 };
 
 /* Set RSS configuration */
 struct rss_cfg_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    hash_bits;
-	u8    tbl_len;
-	u8    tbl_offset;
-#define RSS_IND_TBL_LEN_PER_MBX_MSG	8
-	u8    ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG];
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		hash_bits;
+	uint8_t		tbl_len;
+	uint8_t		tbl_offset;
+#define	RSS_IND_TBL_LEN_PER_MBX_MSG	8
+	uint8_t		ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG];
 };
 
 struct bgx_stats_msg {
-	u8    msg;
-	u8    vf_id;
-	u8    rx;
-	u8    idx;
-	u64   stats;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	uint8_t		rx;
+	uint8_t		idx;
+	uint64_t	stats;
 };
 
 /* Physical interface link status */
 struct bgx_link_status {
-	u8    msg;
-	u8    link_up;
-	u8    duplex;
-	u32   speed;
-};
-
-#ifdef VNIC_MULTI_QSET_SUPPORT
-/* Get Extra Qset IDs */
-struct sqs_alloc {
-	u8    msg;
-	u8    vf_id;
-	u8    qs_count;
-};
-
-struct nicvf_ptr {
-	u8    msg;
-	u8    vf_id;
-	bool  sqs_mode;
-	u8    sqs_id;
-	u64   nicvf;
+	uint8_t		msg;
+	uint8_t		link_up;
+	uint8_t		duplex;
+	uint32_t	speed;
 };
-#endif
 
 /* Set interface in loopback mode */
 struct set_loopback {
-	u8    msg;
-	u8    vf_id;
-	bool  enable;
+	uint8_t		msg;
+	uint8_t		vf_id;
+	boolean_t	enable;
 };
 
 /* 128 bit shared memory between PF and each VF */
 union nic_mbx {
-	struct { u8 msg; }	msg;
+	struct {
+		uint8_t msg;
+	} msg;
 	struct nic_cfg_msg	nic_cfg;
 	struct qs_cfg_msg	qs;
 	struct rq_cfg_msg	rq;
@@ -507,33 +466,23 @@ union nic_mbx {
 	struct cpi_cfg_msg	cpi_cfg;
 	struct rss_sz_msg	rss_size;
 	struct rss_cfg_msg	rss_cfg;
-	struct bgx_stats_msg    bgx_stats;
-	struct bgx_link_status  link_status;
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	struct sqs_alloc        sqs_alloc;
-	struct nicvf_ptr	nicvf;
-#endif
+	struct bgx_stats_msg	bgx_stats;
+	struct bgx_link_status	link_status;
 	struct set_loopback	lbk;
 };
 
-#define NIC_NODE_ID_MASK	0x03
-#define NIC_NODE_ID_SHIFT	44
+#define	NIC_NODE_ID_MASK	0x03
+#define	NIC_NODE_ID_SHIFT	44
 
-static inline int nic_get_node_id(struct pci_dev *pdev)
+static __inline int
+nic_get_node_id(struct resource *res)
 {
-	u64 addr = pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM);
+	pci_addr_t addr;
+
+	addr = rman_get_start(res);
 	return ((addr >> NIC_NODE_ID_SHIFT) & NIC_NODE_ID_MASK);
 }
 
-int nicvf_set_real_num_queues(struct net_device *netdev,
-			      int tx_queues, int rx_queues);
-int nicvf_open(struct net_device *netdev);
-int nicvf_stop(struct net_device *netdev);
 int nicvf_send_msg_to_pf(struct nicvf *vf, union nic_mbx *mbx);
-void nicvf_config_rss(struct nicvf *nic);
-void nicvf_set_rss_key(struct nicvf *nic);
-void nicvf_set_ethtool_ops(struct net_device *netdev);
-void nicvf_update_stats(struct nicvf *nic);
-void nicvf_update_lmac_stats(struct nicvf *nic);
 
 #endif /* NIC_H */

Modified: head/sys/dev/vnic/nic_main.c
==============================================================================
--- head/sys/dev/vnic/nic_main.c	Sun Oct 18 21:39:15 2015	(r289550)
+++ head/sys/dev/vnic/nic_main.c	Sun Oct 18 22:02:58 2015	(r289551)
@@ -27,135 +27,411 @@
  *
  */
 
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include <linux/etherdevice.h>
-#include <linux/of.h>
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
 
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bitset.h>
+#include <sys/bitstring.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/pciio.h>
+#include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/socket.h>
+#include <sys/sockio.h>
+#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+
+#include <net/ethernet.h>
+#include <net/if.h>
+#include <net/if_media.h>
+
+#include <machine/bus.h>
+#include <machine/_inttypes.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <sys/dnv.h>
+#include <sys/nv.h>
+#ifdef PCI_IOV
+#include <sys/iov_schema.h>
+#include <dev/pci/pci_iov.h>
+#endif
+
+#include "thunder_bgx.h"
 #include "nic_reg.h"
 #include "nic.h"
 #include "q_struct.h"
-#include "thunder_bgx.h"
 
-#define DRV_NAME	"thunder-nic"
-#define DRV_VERSION	"1.0"
+#define	VNIC_PF_DEVSTR		"Cavium Thunder NIC Physical Function Driver"
+
+#define	VNIC_PF_REG_RID		PCIR_BAR(PCI_CFG_REG_BAR_NUM)
+
+#define	NIC_SET_VF_LMAC_MAP(bgx, lmac)		((((bgx) & 0xF) << 4) | ((lmac) & 0xF))
+#define	NIC_GET_BGX_FROM_VF_LMAC_MAP(map)	(((map) >> 4) & 0xF)
+#define	NIC_GET_LMAC_FROM_VF_LMAC_MAP(map)	((map) & 0xF)
+
+/* Structure to be used by the SR-IOV for VF configuration schemas */
+struct nicvf_info {
+	boolean_t		vf_enabled;
+	int			vf_flags;
+};
 
 struct nicpf {
-	struct pci_dev		*pdev;
-	u8			rev_id;
-	u8			node;
-	unsigned int		flags;
-	u8			num_vf_en;      /* No of VF enabled */
-	bool			vf_enabled[MAX_NUM_VFS_SUPPORTED];
-	void __iomem		*reg_base;       /* Register start address */
-#ifdef VNIC_MULTI_QSET_SUPPORT
-	u8			num_sqs_en;	/* Secondary qsets enabled */
-	u64			nicvf[MAX_NUM_VFS_SUPPORTED];
-	u8			vf_sqs[MAX_NUM_VFS_SUPPORTED][MAX_SQS_PER_VF];
-	u8			pqs_vf[MAX_NUM_VFS_SUPPORTED];
-	bool			sqs_used[MAX_NUM_VFS_SUPPORTED];
-#endif
+	device_t		dev;
+	uint8_t			rev_id;
+	uint8_t			node;
+	u_int			flags;
+	uint8_t			num_vf_en;      /* No of VF enabled */
+	struct nicvf_info	vf_info[MAX_NUM_VFS_SUPPORTED];
+	struct resource *	reg_base;       /* Register start address */
 	struct pkind_cfg	pkind;
-#define	NIC_SET_VF_LMAC_MAP(bgx, lmac)	(((bgx & 0xF) << 4) | (lmac & 0xF))
-#define	NIC_GET_BGX_FROM_VF_LMAC_MAP(map)	((map >> 4) & 0xF)
-#define	NIC_GET_LMAC_FROM_VF_LMAC_MAP(map)	(map & 0xF)
-	u8			vf_lmac_map[MAX_LMAC];
-	struct delayed_work     dwork;
-	struct workqueue_struct *check_link;
-	u8			link[MAX_LMAC];
-	u8			duplex[MAX_LMAC];
-	u32			speed[MAX_LMAC];
-	u16			cpi_base[MAX_NUM_VFS_SUPPORTED];
-	u16			rss_ind_tbl_size;
-	bool			mbx_lock[MAX_NUM_VFS_SUPPORTED];
+	uint8_t			vf_lmac_map[MAX_LMAC];
+	boolean_t		mbx_lock[MAX_NUM_VFS_SUPPORTED];
+
+	struct callout		check_link;
+	struct mtx		check_link_mtx;
+
+	uint8_t			link[MAX_LMAC];
+	uint8_t			duplex[MAX_LMAC];
+	uint32_t		speed[MAX_LMAC];
+	uint16_t		cpi_base[MAX_NUM_VFS_SUPPORTED];
+	uint16_t		rss_ind_tbl_size;
 
 	/* MSI-X */
-	bool			msix_enabled;
-	u8			num_vec;
+	boolean_t		msix_enabled;
+	uint8_t			num_vec;
 	struct msix_entry	msix_entries[NIC_PF_MSIX_VECTORS];
-	bool			irq_allocated[NIC_PF_MSIX_VECTORS];
+	struct resource *	msix_table_res;
 };
 
-/* Supported devices */
-static const struct pci_device_id nic_id_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_NIC_PF) },
-	{ 0, }  /* end of table */
+static int nicpf_probe(device_t);
+static int nicpf_attach(device_t);
+static int nicpf_detach(device_t);
+
+#ifdef PCI_IOV
+static int nicpf_iov_init(device_t, uint16_t, const nvlist_t *);
+static void nicpf_iov_uninit(device_t);
+static int nicpf_iov_addr_vf(device_t, uint16_t, const nvlist_t *);
+#endif
+
+static device_method_t nicpf_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		nicpf_probe),
+	DEVMETHOD(device_attach,	nicpf_attach),
+	DEVMETHOD(device_detach,	nicpf_detach),
+	/* PCI SR-IOV interface */
+#ifdef PCI_IOV
+	DEVMETHOD(pci_iov_init,		nicpf_iov_init),
+	DEVMETHOD(pci_iov_uninit,	nicpf_iov_uninit),
+	DEVMETHOD(pci_iov_add_vf,	nicpf_iov_addr_vf),
+#endif
+	DEVMETHOD_END,
 };
 
-MODULE_AUTHOR("Sunil Goutham");
-MODULE_DESCRIPTION("Cavium Thunder NIC Physical Function Driver");
-MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, nic_id_table);
-
-/* The Cavium ThunderX network controller can *only* be found in SoCs
- * containing the ThunderX ARM64 CPU implementation.  All accesses to the device
- * registers on this platform are implicitly strongly ordered with respect
- * to memory accesses. So writeq_relaxed() and readq_relaxed() are safe to use
- * with no memory barriers in this driver.  The readq()/writeq() functions add
- * explicit ordering operation which in this case are redundant, and only
- * add overhead.
+static driver_t nicpf_driver = {
+	"vnicpf",
+	nicpf_methods,
+	sizeof(struct nicpf),
+};
+
+static devclass_t nicpf_devclass;
+
+DRIVER_MODULE(nicpf, pci, nicpf_driver, nicpf_devclass, 0, 0);
+MODULE_DEPEND(nicpf, pci, 1, 1, 1);
+MODULE_DEPEND(nicpf, ether, 1, 1, 1);
+MODULE_DEPEND(nicpf, thunder_bgx, 1, 1, 1);
+
+static int nicpf_alloc_res(struct nicpf *);
+static void nicpf_free_res(struct nicpf *);
+static void nic_set_lmac_vf_mapping(struct nicpf *);
+static void nic_init_hw(struct nicpf *);
+static int nic_sriov_init(device_t, struct nicpf *);
+static void nic_poll_for_link(void *);
+static int nic_register_interrupts(struct nicpf *);
+static void nic_unregister_interrupts(struct nicpf *);
+
+/*
+ * Device interface
+ */
+static int
+nicpf_probe(device_t dev)
+{
+	uint16_t vendor_id;
+	uint16_t device_id;
+
+	vendor_id = pci_get_vendor(dev);
+	device_id = pci_get_device(dev);
+
+	if (vendor_id == PCI_VENDOR_ID_CAVIUM &&
+	    device_id == PCI_DEVICE_ID_THUNDER_NIC_PF) {
+		device_set_desc(dev, VNIC_PF_DEVSTR);
+		return (BUS_PROBE_DEFAULT);
+	}
+
+	return (ENXIO);
+}
+
+static int
+nicpf_attach(device_t dev)
+{
+	struct nicpf *nic;
+	int err;
+
+	nic = device_get_softc(dev);
+	nic->dev = dev;
+
+	/* Enable bus mastering */
+	pci_enable_busmaster(dev);
+
+	/* Allocate PCI resources */
+	err = nicpf_alloc_res(nic);
+	if (err != 0) {
+		device_printf(dev, "Could not allocate PCI resources\n");
+		return (err);
+	}
+
+	nic->node = nic_get_node_id(nic->reg_base);
+	nic->rev_id = pci_read_config(dev, PCIR_REVID, 1);
+
+	/* Enable Traffic Network Switch (TNS) bypass mode by default */
+	nic->flags &= ~NIC_TNS_ENABLED;
+	nic_set_lmac_vf_mapping(nic);
+
+	/* Initialize hardware */
+	nic_init_hw(nic);
+
+	/* Set RSS TBL size for each VF */
+	nic->rss_ind_tbl_size = NIC_MAX_RSS_IDR_TBL_SIZE;
+
+	/* Setup interrupts */
+	err = nic_register_interrupts(nic);
+	if (err != 0)
+		goto err_free_res;
+
+	/* Configure SRIOV */
+	err = nic_sriov_init(dev, nic);
+	if (err != 0)
+		goto err_free_intr;
+
+	if (nic->flags & NIC_TNS_ENABLED)
+		return (0);
+
+	mtx_init(&nic->check_link_mtx, "VNIC PF link poll", NULL, MTX_DEF);
+	/* Register physical link status poll callout */
+	callout_init_mtx(&nic->check_link, &nic->check_link_mtx, 0);
+	mtx_lock(&nic->check_link_mtx);
+	nic_poll_for_link(nic);
+	mtx_unlock(&nic->check_link_mtx);
+
+	return (0);
+
+err_free_intr:
+	nic_unregister_interrupts(nic);
+err_free_res:
+	nicpf_free_res(nic);
+	pci_disable_busmaster(dev);
+
+	return (err);
+}
+
+static int
+nicpf_detach(device_t dev)

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201510182202.t9IM2wL0025859>