Date: Tue, 13 Oct 2015 03:12:12 +0000 (UTC) From: "Conrad E. Meyer" <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r289208 - in head/sys/dev/ntb: if_ntb ntb_hw Message-ID: <201510130312.t9D3CCm2046612@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Tue Oct 13 03:12:11 2015 New Revision: 289208 URL: https://svnweb.freebsd.org/changeset/base/289208 Log: NTB: MFV 948d3a65: Xeon Errata Workaround There is a Xeon hardware errata related to writes to SDOORBELL or B2BDOORBELL in conjunction with inbound access to NTB MMIO Space, which may hang the system. To workaround this issue, use one of the memory windows to access the interrupt and scratch pad registers on the remote system. This bypasses the issue, but removes one of the memory windows from use by the transport. This reduction of MWs necessitates adding some logic to determine the number of available MWs. Since some NTB usage methodologies may have unidirectional traffic, the ability to disable the workaround via modparm has been added. See BF113 in http://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/xeon-c5500-c3500-spec-update.pdf See BT119 in http://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/xeon-e5-family-spec-update.pdf Authored by: Jon Mason Obtained from: Linux Sponsored by: EMC / Isilon Storage Division Modified: head/sys/dev/ntb/if_ntb/if_ntb.c head/sys/dev/ntb/ntb_hw/ntb_hw.c head/sys/dev/ntb/ntb_hw/ntb_hw.h head/sys/dev/ntb/ntb_hw/ntb_regs.h Modified: head/sys/dev/ntb/if_ntb/if_ntb.c ============================================================================== --- head/sys/dev/ntb/if_ntb/if_ntb.c Tue Oct 13 03:11:21 2015 (r289207) +++ head/sys/dev/ntb/if_ntb/if_ntb.c Tue Oct 13 03:12:11 2015 (r289208) @@ -78,6 +78,11 @@ __FBSDID("$FreeBSD$"); #define NTB_RXQ_SIZE 300 static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN; + +/* + * This is an oversimplification to work around Xeon Errata. The second client + * may be usable for unidirectional traffic. + */ static unsigned int max_num_clients = 1; STAILQ_HEAD(ntb_queue_list, ntb_queue_entry); @@ -1032,11 +1037,16 @@ ntb_transport_link_work(void *arg) struct ntb_softc *ntb = nt->ntb; struct ntb_transport_qp *qp; uint64_t val64; - uint32_t val; - int rc, i; + uint32_t val, i, num_mw; + int rc; + + if (ntb_has_feature(ntb, NTB_REGS_THRU_MW)) + num_mw = NTB_NUM_MW - 1; + else + num_mw = NTB_NUM_MW; /* send the local info, in the opposite order of the way we read it */ - for (i = 0; i < NTB_NUM_MW; i++) { + for (i = 0; i < num_mw; i++) { rc = ntb_write_remote_spad(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), ntb_get_mw_size(ntb, i) >> 32); if (rc != 0) @@ -1048,7 +1058,7 @@ ntb_transport_link_work(void *arg) goto out; } - rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_MWS, NTB_NUM_MW); + rc = ntb_write_remote_spad(ntb, IF_NTB_NUM_MWS, num_mw); if (rc != 0) goto out; @@ -1079,10 +1089,10 @@ ntb_transport_link_work(void *arg) if (rc != 0) goto out; - if (val != NTB_NUM_MW) + if (val != num_mw) goto out; - for (i = 0; i < NTB_NUM_MW; i++) { + for (i = 0; i < num_mw; i++) { rc = ntb_read_local_spad(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), &val); if (rc != 0) Modified: head/sys/dev/ntb/ntb_hw/ntb_hw.c ============================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw.c Tue Oct 13 03:11:21 2015 (r289207) +++ head/sys/dev/ntb/ntb_hw/ntb_hw.c Tue Oct 13 03:12:11 2015 (r289208) @@ -81,9 +81,6 @@ enum ntb_device_type { #define HAS_FEATURE(feature) \ ((ntb->features & (feature)) != 0) -#define NTB_BAR_SIZE_4K (1 << 0) -#define NTB_REGS_THRU_MW (1 << 1) - struct ntb_hw_info { uint32_t device_id; const char *desc; @@ -131,9 +128,9 @@ struct ntb_softc { struct ntb_db_cb *db_cb; struct { - uint32_t max_spads; - uint32_t max_db_bits; - uint32_t msix_cnt; + uint8_t max_spads; + uint8_t max_db_bits; + uint8_t msix_cnt; } limits; struct { uint32_t pdb; @@ -706,6 +703,28 @@ ntb_setup_xeon(struct ntb_softc *ntb) ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; + /* + * There is a Xeon hardware errata related to writes to SDOORBELL or + * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, + * which may hang the system. To workaround this use the second memory + * window to access the interrupt and scratch pad registers on the + * remote system. + */ + if (HAS_FEATURE(NTB_REGS_THRU_MW)) + /* + * Set the Limit register to 4k, the minimum size, to prevent + * an illegal access. + */ + ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, + ntb_get_mw_size(ntb, 1) + 0x1000); + else + /* + * Disable the limit register, just in case it is set to + * something silly. + */ + ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); + + if (ntb->conn_type == NTB_CONN_B2B) { ntb->reg_ofs.sdb = XEON_B2B_DOORBELL_OFFSET; ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; @@ -825,9 +844,18 @@ configure_xeon_secondary_side_bars(struc if (HAS_FEATURE(NTB_REGS_THRU_MW)) ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, MBAR01_DSD_ADDR); - else + else { ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, PBAR4XLAT_USD_ADDR); + /* + * B2B_XLAT_OFFSET is a 64-bit register but can only be + * written 32 bits at a time. + */ + ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, + MBAR01_DSD_ADDR & 0xffffffff); + ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, + MBAR01_DSD_ADDR >> 32); + } ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_USD_ADDR); ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_USD_ADDR); ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_USD_ADDR); @@ -836,9 +864,18 @@ configure_xeon_secondary_side_bars(struc if (HAS_FEATURE(NTB_REGS_THRU_MW)) ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, MBAR01_USD_ADDR); - else + else { ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, PBAR4XLAT_DSD_ADDR); + /* + * B2B_XLAT_OFFSET is a 64-bit register but can only be + * written 32 bits at a time. + */ + ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, + MBAR01_USD_ADDR & 0xffffffff); + ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, + MBAR01_USD_ADDR >> 32); + } ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, MBAR01_DSD_ADDR); ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, MBAR23_DSD_ADDR); ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, MBAR45_DSD_ADDR); @@ -1171,7 +1208,7 @@ ntb_unregister_transport(struct ntb_soft * * RETURNS: total number of scratch pad registers available */ -int +uint8_t ntb_get_max_spads(struct ntb_softc *ntb) { @@ -1415,3 +1452,11 @@ device_t ntb_get_device(struct ntb_softc return (ntb->device); } + +/* Export HW-specific errata information. */ +bool +ntb_has_feature(struct ntb_softc *ntb, uint64_t feature) +{ + + return (HAS_FEATURE(feature)); +} Modified: head/sys/dev/ntb/ntb_hw/ntb_hw.h ============================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw.h Tue Oct 13 03:11:21 2015 (r289207) +++ head/sys/dev/ntb/ntb_hw/ntb_hw.h Tue Oct 13 03:12:11 2015 (r289208) @@ -58,7 +58,7 @@ void *ntb_find_transport(struct ntb_soft struct ntb_softc *ntb_register_transport(struct ntb_softc *ntb, void *transport); void ntb_unregister_transport(struct ntb_softc *ntb); -int ntb_get_max_spads(struct ntb_softc *ntb); +uint8_t ntb_get_max_spads(struct ntb_softc *ntb); int ntb_write_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t val); int ntb_read_local_spad(struct ntb_softc *ntb, unsigned int idx, uint32_t *val); int ntb_write_remote_spad(struct ntb_softc *ntb, unsigned int idx, @@ -73,4 +73,8 @@ void ntb_ring_sdb(struct ntb_softc *ntb, bool ntb_query_link_status(struct ntb_softc *ntb); device_t ntb_get_device(struct ntb_softc *ntb); +#define NTB_BAR_SIZE_4K (1 << 0) +#define NTB_REGS_THRU_MW (1 << 1) +bool ntb_has_feature(struct ntb_softc *, uint64_t); + #endif /* _NTB_HW_H_ */ Modified: head/sys/dev/ntb/ntb_hw/ntb_regs.h ============================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_regs.h Tue Oct 13 03:11:21 2015 (r289207) +++ head/sys/dev/ntb/ntb_hw/ntb_regs.h Tue Oct 13 03:12:11 2015 (r289208) @@ -71,7 +71,8 @@ #define XEON_WCCNTRL_OFFSET 0x00e0 #define XEON_B2B_SPAD_OFFSET 0x0100 #define XEON_B2B_DOORBELL_OFFSET 0x0140 -#define XEON_B2B_XLAT_OFFSET 0x0144 +#define XEON_B2B_XLAT_OFFSETL 0x0144 +#define XEON_B2B_XLAT_OFFSETU 0x0148 #define SOC_MSIX_CNT 34 #define SOC_MAX_SPADS 16 @@ -136,16 +137,16 @@ #define NTB_DEV_DSD 1 #define NTB_DEV_USD 0 -#define PBAR2XLAT_USD_ADDR 0x0000004000000000 -#define PBAR4XLAT_USD_ADDR 0x0000008000000000 -#define MBAR01_USD_ADDR 0x000000210000000c -#define MBAR23_USD_ADDR 0x000000410000000c -#define MBAR45_USD_ADDR 0x000000810000000c -#define PBAR2XLAT_DSD_ADDR 0x0000004100000000 -#define PBAR4XLAT_DSD_ADDR 0x0000008100000000 -#define MBAR01_DSD_ADDR 0x000000200000000c -#define MBAR23_DSD_ADDR 0x000000400000000c -#define MBAR45_DSD_ADDR 0x000000800000000c +#define PBAR2XLAT_USD_ADDR 0x0000004000000000ull +#define PBAR4XLAT_USD_ADDR 0x0000008000000000ull +#define MBAR01_USD_ADDR 0x000000210000000cull +#define MBAR23_USD_ADDR 0x000000410000000cull +#define MBAR45_USD_ADDR 0x000000810000000cull +#define PBAR2XLAT_DSD_ADDR 0x0000004100000000ull +#define PBAR4XLAT_DSD_ADDR 0x0000008100000000ull +#define MBAR01_DSD_ADDR 0x000000200000000cull +#define MBAR23_DSD_ADDR 0x000000400000000cull +#define MBAR45_DSD_ADDR 0x000000800000000cull /* XEON Shadowed MMIO Space */ #define XEON_SHADOW_PDOORBELL_OFFSET 0x60
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201510130312.t9D3CCm2046612>